ExceptionWrapper.h \
Executor.h \
EvictingCacheMap.h \
+ experimental/AsymmetricMemoryBarrier.h \
experimental/AutoTimer.h \
experimental/Bits.h \
experimental/BitVectorCoding.h \
portability/String.h \
portability/Syslog.h \
portability/SysFile.h \
+ portability/SysMembarrier.h \
portability/SysMman.h \
portability/SysResource.h \
portability/SysStat.h \
portability/Stdlib.cpp \
portability/String.cpp \
portability/SysFile.cpp \
+ portability/SysMembarrier.cpp \
portability/SysMman.cpp \
portability/SysResource.cpp \
portability/SysStat.cpp \
TimeoutQueue.cpp \
Uri.cpp \
Version.cpp \
+ experimental/AsymmetricMemoryBarrier.cpp \
experimental/bser/Dump.cpp \
experimental/bser/Load.cpp \
experimental/DynamicParser.cpp \
// we will take the next one.
#define FOLLY_STATIC_CTOR_PRIORITY_MAX __attribute__((__init_priority__(102)))
#endif
+
+namespace folly {
+
+#ifdef __linux__
+constexpr auto kIsLinux = true;
+#else
+constexpr auto kIsLinux = false;
+#endif
+}
--- /dev/null
+/*
+ * Copyright 2016 Facebook, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "AsymmetricMemoryBarrier.h"
+
+#include <folly/Exception.h>
+#include <folly/Indestructible.h>
+#include <folly/portability/SysMembarrier.h>
+#include <folly/portability/SysMman.h>
+#include <mutex>
+
+namespace folly {
+
+namespace {
+
+struct DummyPageCreator {
+ DummyPageCreator() {
+ get();
+ }
+
+ static void* get() {
+ static auto ptr =
+ kIsLinux && !detail::sysMembarrierAvailable() ? create() : nullptr;
+ return ptr;
+ }
+
+ private:
+ static void* create() {
+ auto ptr = mmap(nullptr, 1, PROT_READ, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+ checkUnixError(reinterpret_cast<uintptr_t>(ptr), "mmap");
+
+ // Lock the memory so it can't get paged out. If it gets paged out, changing
+ // its protection won't accomplish anything.
+ auto r = mlock(ptr, 1);
+ checkUnixError(r, "mlock");
+
+ return ptr;
+ }
+};
+
+// Make sure dummy page is always initialized before shutdown
+DummyPageCreator dummyPageCreator;
+
+void mprotectMembarrier() {
+ auto dummyPage = dummyPageCreator.get();
+
+ // This function is required to be safe to call on shutdown,
+ // so we must leak the mutex.
+ static Indestructible<std::mutex> mprotectMutex;
+ std::lock_guard<std::mutex> lg(*mprotectMutex);
+
+ int r = 0;
+ r = mprotect(dummyPage, 1, PROT_READ | PROT_WRITE);
+ checkUnixError(r, "mprotect");
+
+ r = mprotect(dummyPage, 1, PROT_READ);
+ checkUnixError(r, "mprotect");
+}
+}
+
+void asymmetricHeavyBarrier() {
+ if (kIsLinux) {
+ static const bool useSysMembarrier = detail::sysMembarrierAvailable();
+ if (useSysMembarrier) {
+ auto r = detail::sysMembarrier();
+ checkUnixError(r, "membarrier");
+ } else {
+ mprotectMembarrier();
+ }
+ } else {
+ std::atomic_thread_fence(std::memory_order_seq_cst);
+ }
+}
+}
--- /dev/null
+/*
+ * Copyright 2016 Facebook, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <atomic>
+
+#include <folly/portability/Asm.h>
+
+namespace folly {
+
+FOLLY_ALWAYS_INLINE void asymmetricLightBarrier() {
+ if (kIsLinux) {
+ asm_volatile_memory();
+ } else {
+ std::atomic_thread_fence(std::memory_order_seq_cst);
+ }
+}
+
+void asymmetricHeavyBarrier();
+}
#pragma once
#include <folly/ThreadLocal.h>
+#include <folly/experimental/AsymmetricMemoryBarrier.h>
namespace folly {
state_ = State::GLOBAL_TRANSITION;
+ asymmetricHeavyBarrier();
+
std::weak_ptr<void> collectGuardWeak = collectGuard_;
// Make sure we can't create new LocalRefCounts
return false;
}
- auto count = count_ += delta;
+ // This is equivalent to atomic fetch_add. We know that this operation
+ // is always performed from a single thread. asymmetricLightBarrier()
+ // makes things faster than atomic fetch_add on platforms with native
+ // support.
+ auto count = count_.load(std::memory_order_relaxed) + delta;
+ count_.store(count, std::memory_order_relaxed);
+
+ asymmetricLightBarrier();
if (UNLIKELY(refCount_.state_.load() != State::LOCAL)) {
std::lock_guard<std::mutex> lg(collectMutex_);
--- /dev/null
+/*
+ * Copyright 2016 Facebook, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "SysMembarrier.h"
+
+#include <mutex>
+
+#include <folly/Portability.h>
+#include <folly/portability/Unistd.h>
+
+#if !defined(__NR_membarrier) && defined(FOLLY_X64)
+#define __NR_membarrier 324
+#define MEMBARRIER_CMD_QUERY 0
+#define MEMBARRIER_CMD_SHARED 1
+#endif
+
+namespace folly {
+namespace detail {
+
+bool sysMembarrierAvailable() {
+ if (!kIsLinux) {
+ return false;
+ }
+
+#ifdef __NR_membarrier
+ auto r = syscall(__NR_membarrier, MEMBARRIER_CMD_QUERY, /* flags = */ 0);
+ if (r == -1) {
+ return false;
+ }
+
+ return r & MEMBARRIER_CMD_SHARED;
+#else
+ return false;
+#endif
+}
+
+int sysMembarrier() {
+#ifdef __NR_membarrier
+ return syscall(__NR_membarrier, MEMBARRIER_CMD_SHARED, /* flags = */ 0);
+#else
+ return -1;
+#endif
+}
+}
+}
--- /dev/null
+/*
+ * Copyright 2016 Facebook, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+namespace folly {
+namespace detail {
+
+int sysMembarrier();
+bool sysMembarrierAvailable();
+}
+}
projects / folly.git / commitdiff