/*
- * Copyright 2016 Facebook, Inc.
+ * Copyright 2015-present Facebook, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
#pragma once
#include <stdint.h>
+
#include <atomic>
#include <thread>
#include <type_traits>
+
#include <folly/Likely.h>
-#include <folly/detail/CacheLocality.h>
+#include <folly/concurrency/CacheLocality.h>
#include <folly/detail/Futex.h>
#include <folly/portability/Asm.h>
#include <folly/portability/SysResource.h>
uint16_t slot_;
};
-template <bool ReaderPriority,
- typename Tag_ = void,
- template <typename> class Atom = std::atomic,
- bool BlockImmediately = false>
+template <
+ bool ReaderPriority,
+ typename Tag_ = void,
+ template <typename> class Atom = std::atomic,
+ bool BlockImmediately = false>
class SharedMutexImpl {
public:
static constexpr bool kReaderPriority = ReaderPriority;
class UpgradeHolder;
class WriteHolder;
- constexpr SharedMutexImpl() : state_(0) {}
+ constexpr SharedMutexImpl() noexcept : state_(0) {}
SharedMutexImpl(const SharedMutexImpl&) = delete;
SharedMutexImpl(SharedMutexImpl&&) = delete;
};
// 32 bits of state
- Futex state_;
+ Futex state_{};
// S count needs to be on the end, because we explicitly allow it to
// underflow. This can occur while we are in the middle of applying
typedef Atom<uintptr_t> DeferredReaderSlot;
private:
- FOLLY_ALIGN_TO_AVOID_FALSE_SHARING static DeferredReaderSlot deferredReaders
- [kMaxDeferredReaders *
- kDeferredSeparationFactor];
+ alignas(hardware_destructive_interference_size) static DeferredReaderSlot
+ deferredReaders[kMaxDeferredReaders * kDeferredSeparationFactor];
// Performs an exclusive lock, waiting for state_ & waitMask to be
// zero first
}
uint32_t after = (state & kMayDefer) == 0 ? 0 : kPrevDefer;
- if (!ReaderPriority || (state & (kMayDefer | kHasS)) == 0) {
+ if (!kReaderPriority || (state & (kMayDefer | kHasS)) == 0) {
// Block readers immediately, either because we are in write
// priority mode or because we can acquire the lock in one
// step. Note that if state has kHasU, then we are doing an
return false;
}
- if (ReaderPriority && (state & kHasE) == 0) {
+ if (kReaderPriority && (state & kHasE) == 0) {
assert((state & kBegunE) != 0);
if (!state_.compare_exchange_strong(state,
(state & ~kBegunE) | kHasE)) {
WaitContext& ctx) {
#ifdef RUSAGE_THREAD
struct rusage usage;
+ std::memset(&usage, 0, sizeof(usage));
long before = -1;
#endif
for (uint32_t yieldCount = 0; yieldCount < kMaxSoftYieldCount;
return;
}
}
- asm_pause();
+ asm_volatile_pause();
if (UNLIKELY(++spinCount >= kMaxSpinCount)) {
applyDeferredReaders(state, ctx, slot);
return;
#ifdef RUSAGE_THREAD
struct rusage usage;
+ std::memset(&usage, 0, sizeof(usage));
long before = -1;
#endif
for (uint32_t yieldCount = 0; yieldCount < kMaxSoftYieldCount;
public:
class ReadHolder {
- public:
ReadHolder() : lock_(nullptr) {}
- explicit ReadHolder(const SharedMutexImpl* lock) : ReadHolder(*lock) {}
+ public:
+ explicit ReadHolder(const SharedMutexImpl* lock)
+ : lock_(const_cast<SharedMutexImpl*>(lock)) {
+ if (lock_) {
+ lock_->lock_shared(token_);
+ }
+ }
explicit ReadHolder(const SharedMutexImpl& lock)
: lock_(const_cast<SharedMutexImpl*>(&lock)) {
};
class UpgradeHolder {
- public:
UpgradeHolder() : lock_(nullptr) {}
- explicit UpgradeHolder(SharedMutexImpl* lock) : UpgradeHolder(*lock) {}
+ public:
+ explicit UpgradeHolder(SharedMutexImpl* lock) : lock_(lock) {
+ if (lock_) {
+ lock_->lock_upgrade();
+ }
+ }
explicit UpgradeHolder(SharedMutexImpl& lock) : lock_(&lock) {
lock_->lock_upgrade();
};
class WriteHolder {
- public:
WriteHolder() : lock_(nullptr) {}
- explicit WriteHolder(SharedMutexImpl* lock) : WriteHolder(*lock) {}
+ public:
+ explicit WriteHolder(SharedMutexImpl* lock) : lock_(lock) {
+ if (lock_) {
+ lock_->lock();
+ }
+ }
explicit WriteHolder(SharedMutexImpl& lock) : lock_(&lock) {
lock_->lock();
typename Tag_,
template <typename> class Atom,
bool BlockImmediately>
-typename SharedMutexImpl<ReaderPriority, Tag_, Atom, BlockImmediately>::
- DeferredReaderSlot
- SharedMutexImpl<ReaderPriority, Tag_, Atom, BlockImmediately>::
- deferredReaders[kMaxDeferredReaders * kDeferredSeparationFactor] =
- {};
+alignas(hardware_destructive_interference_size)
+ typename SharedMutexImpl<ReaderPriority, Tag_, Atom, BlockImmediately>::
+ DeferredReaderSlot
+ SharedMutexImpl<ReaderPriority, Tag_, Atom, BlockImmediately>::
+ deferredReaders[kMaxDeferredReaders * kDeferredSeparationFactor] = {};
template <
bool ReaderPriority,
// starting point for our empty-slot search, can change after
// calling waitForZeroBits
uint32_t bestSlot =
- (uint32_t)folly::detail::AccessSpreader<Atom>::current(
- kMaxDeferredReaders);
+ (uint32_t)folly::AccessSpreader<Atom>::current(kMaxDeferredReaders);
// deferred readers are already enabled, or it is time to
// enable them if we can find a slot