Math.h \
Memory.h \
MemoryMapping.h \
+ memory/UninitializedMemoryHacks.h \
MicroSpinLock.h \
MicroLock.h \
MoveWrapper.h \
portability/SysUio.h \
portability/Time.h \
portability/TypeTraits.h \
- portability/Windows.h \
portability/Unistd.h \
+ portability/Windows.h \
Preprocessor.h \
PriorityMPMCQueue.h \
ProducerConsumerQueue.h \
--- /dev/null
+/*
+ * Copyright 2017 Facebook, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <string>
+#include <type_traits>
+#include <vector>
+
+namespace {
+// This struct is different in every translation unit. We use template
+// instantiations to define inline freestanding methods. Since the
+// methods are inline it is fine to define them in multiple translation
+// units, but the instantiation itself would be an ODR violation if it is
+// present in the program more than once. By tagging the instantiations
+// with this struct, we avoid ODR problems for the instantiation while
+// allowing the resulting methods to be inline-able. If you think that
+// seems hacky keep reading...
+struct FollyMemoryDetailTranslationUnitTag {};
+} // anon namespace
+namespace folly {
+namespace detail {
+void unsafeStringSetLargerSize(std::string& s, std::size_t n);
+template <typename T>
+void unsafeVectorSetLargerSize(std::vector<T>& v, std::size_t n);
+} // namespace detail
+
+/*
+ * This file provides helper functions resizeWithoutInitialization()
+ * that can resize std::string or std::vector without constructing or
+ * initializing new elements.
+ *
+ * IMPORTANT: These functions can be unsafe if used improperly. If you
+ * don't write to an element with index >= oldSize and < newSize, reading
+ * the element can expose arbitrary memory contents to the world, including
+ * the contents of old strings. Use ASAN in your tests, and pay extra
+ * attention to failure paths.
+ *
+ * You should only use this if you have profiling data from production
+ * that shows that this is not a premature optimization. This code is
+ * designed for retroactively optimizing code where touching every element
+ * twice (or touching never-used elements once) shows up in profiling,
+ * and where restructuring the code to use fixed-length arrays or IOBuf-s
+ * would be difficult.
+ *
+ * NOTE: Just because .resize() shows up in your profile (probably
+ * via one of the intrinsic memset implementations) doesn't mean that
+ * these functions will make your program faster. Most of the cost
+ * of memset comes from cache misses, so avoiding the memset means
+ * that the cache miss cost just gets pushed to the following code.
+ * resizeWithoutInitialization can be a win when the contents are bigger
+ * than a cache level, because the second access isn't free in that case.
+ * It can also be a win if the final length of the string or vector isn't
+ * actually known, so the suffix will be chopped off with a second call
+ * to .resize().
+ */
+
+/**
+ * Like calling s.resize(n), but when growing the string does not
+ * initialize new elements. It is undefined behavior to read from
+ * any element added to the string by this method unless it has been
+ * written to by an operation that follows this call.
+ *
+ * IMPORTANT: Read the warning at the top of this header file.
+ */
+inline void resizeWithoutInitialization(std::string& s, std::size_t n) {
+ if (n <= s.size()) {
+ s.resize(n);
+ } else {
+ // careful not to call reserve unless necessary, as it causes
+ // shrink_to_fit on many platforms
+ if (n > s.capacity()) {
+ s.reserve(n);
+ }
+ detail::unsafeStringSetLargerSize(s, n);
+ }
+}
+
+/**
+ * Like calling v.resize(n), but when growing the vector does not construct
+ * or initialize new elements. It is undefined behavior to read from any
+ * element added to the vector by this method unless it has been written
+ * to by an operation that follows this call.
+ *
+ * Use the FOLLY_DECLARE_VECTOR_RESIZE_WITHOUT_INIT(T) macro to
+ * declare (and inline define) the internals required to call
+ * resizeWithoutInitialization for a std::vector<T>. This must
+ * be done exactly once in each translation unit that wants to call
+ * resizeWithoutInitialization(std::vector<T>&,size_t). char and unsigned
+ * char are provided by default. If you don't do this you will get linker
+ * errors about folly::detail::unsafeVectorSetLargerSize. Requiring that
+ * T be trivially_destructible is only an approximation of the property
+ * required of T. In fact what is required is that any random sequence of
+ * bytes may be safely reinterpreted as a T and passed to T's destructor.
+ *
+ * std::vector<bool> has specialized internals and is not supported.
+ *
+ * IMPORTANT: Read the warning at the top of this header file.
+ */
+template <
+ typename T,
+ typename = typename std::enable_if<
+ std::is_trivially_destructible<T>::value &&
+ !std::is_same<T, bool>::value>::type>
+void resizeWithoutInitialization(std::vector<T>& v, std::size_t n) {
+ if (n <= v.size()) {
+ v.resize(n);
+ } else {
+ if (n > v.capacity()) {
+ v.reserve(n);
+ }
+ detail::unsafeVectorSetLargerSize(v, n);
+ }
+}
+
+namespace detail {
+
+#if defined(_LIBCPP_STRING)
+// libc++
+
+} // namespace detail
+} // namespace folly
+template void std::string::__set_size(std::size_t);
+namespace folly {
+namespace detail {
+
+template <typename Tag, typename T, typename A, A Ptr__set_size>
+struct MakeUnsafeStringSetLargerSize {
+ friend void unsafeStringSetLargerSize(
+ std::basic_string<T>& s,
+ std::size_t n) {
+ // s.__set_size(n);
+ (s.*Ptr__set_size)(n);
+ (&s[0])[n] = '\0';
+ }
+};
+template struct MakeUnsafeStringSetLargerSize<
+ FollyMemoryDetailTranslationUnitTag,
+ char,
+ void (std::string::*)(std::size_t),
+ &std::string::__set_size>;
+
+#elif defined(_GLIBCXX_USE_FB)
+// FBString
+
+template <typename Tag, typename T, typename A, A Ptrstore_>
+struct MakeUnsafeStringSetLargerSize {
+ friend void unsafeStringSetLargerSize(
+ std::basic_string<T>& s,
+ std::size_t n) {
+ // s.store_.expandNoinit(n - s.size(), false);
+ (s.*Ptrstore_).expandNoinit(n - s.size(), false);
+ }
+};
+template struct MakeUnsafeStringSetLargerSize<
+ FollyMemoryDetailTranslationUnitTag,
+ char,
+ std::fbstring_core<char>(std::string::*),
+ &std::string::store_>;
+
+#elif defined(_GLIBCXX_STRING) && _GLIBCXX_USE_CXX11_ABI
+// libstdc++ new implementation with SSO
+
+} // namespace detail
+} // namespace folly
+template void std::string::_M_set_length(std::size_t);
+namespace folly {
+namespace detail {
+
+template <typename Tag, typename T, typename A, A Ptr_M_set_length>
+struct MakeUnsafeStringSetLargerSize {
+ friend void unsafeStringSetLargerSize(
+ std::basic_string<T>& s,
+ std::size_t n) {
+ // s._M_set_length(n);
+ (s.*Ptr_M_set_length)(n);
+ }
+};
+template struct MakeUnsafeStringSetLargerSize<
+ FollyMemoryDetailTranslationUnitTag,
+ char,
+ void (std::string::*)(std::size_t),
+ &std::string::_M_set_length>;
+
+#elif defined(_GLIBCXX_STRING)
+// libstdc++ old implementation
+
+} // namespace detail
+} // namespace folly
+template std::string::_Rep* std::string::_M_rep() const;
+template void std::string::_Rep::_M_set_length_and_sharable(std::size_t);
+namespace folly {
+namespace detail {
+
+template <
+ typename Tag,
+ typename T,
+ typename A,
+ A Ptr_M_rep,
+ typename B,
+ B Ptr_M_set_length_and_sharable>
+struct MakeUnsafeStringSetLargerSize {
+ friend void unsafeStringSetLargerSize(
+ std::basic_string<T>& s,
+ std::size_t n) {
+ // s._M_rep()->_M_set_length_and_sharable(n);
+ auto rep = (s.*Ptr_M_rep)();
+ (rep->*Ptr_M_set_length_and_sharable)(n);
+ }
+};
+template struct MakeUnsafeStringSetLargerSize<
+ FollyMemoryDetailTranslationUnitTag,
+ char,
+ std::string::_Rep* (std::string::*)() const,
+ &std::string::_M_rep,
+ void (std::string::_Rep::*)(std::size_t),
+ &std::string::_Rep::_M_set_length_and_sharable>;
+
+#elif defined(_MSC_VER)
+// MSVC
+
+inline void unsafeStringSetLargerSize(std::string& s, std::size_t n) {
+ s._Eos(n);
+}
+
+#else
+#warn "No implementation for resizeWithoutInitialization of std::string"
+#endif
+
+// This machinery bridges template expansion and macro expansion
+#define FOLLY_DECLARE_VECTOR_RESIZE_WITHOUT_INIT_IMPL(TYPE) \
+ namespace folly { \
+ namespace detail { \
+ void unsafeVectorSetLargerSizeImpl(std::vector<TYPE>& v, std::size_t); \
+ template <> \
+ inline void unsafeVectorSetLargerSize<TYPE>( \
+ std::vector<TYPE> & v, \
+ std::size_t n) { \
+ unsafeVectorSetLargerSizeImpl(v, n); \
+ } \
+ } \
+ }
+
+#if defined(_LIBCPP_VECTOR)
+// libc++
+
+template <typename Tag, typename T, typename A, A Ptr__end_>
+struct MakeUnsafeVectorSetLargerSize {
+ friend void unsafeVectorSetLargerSizeImpl(std::vector<T>& v, std::size_t n) {
+ // v.__end_ += (n - v.size());
+ using Base = std::__vector_base<T, std::allocator<T>>;
+ static_assert(
+ std::is_standard_layout<std::vector<T>>::value &&
+ sizeof(std::vector<T>) == sizeof(Base),
+ "reinterpret_cast safety conditions not met");
+ reinterpret_cast<Base&>(v).*Ptr__end_ += (n - v.size());
+ }
+};
+
+#define FOLLY_DECLARE_VECTOR_RESIZE_WITHOUT_INIT(TYPE) \
+ template struct folly::detail::MakeUnsafeVectorSetLargerSize< \
+ FollyMemoryDetailTranslationUnitTag, \
+ TYPE, \
+ TYPE*(std::__vector_base<TYPE, std::allocator<TYPE>>::*), \
+ &std::vector<TYPE>::__end_>; \
+ FOLLY_DECLARE_VECTOR_RESIZE_WITHOUT_INIT_IMPL(TYPE)
+
+#elif defined(_GLIBCXX_VECTOR)
+// libstdc++
+
+template <
+ typename Tag,
+ typename T,
+ typename A,
+ A Ptr_M_impl,
+ typename B,
+ B Ptr_M_finish>
+struct MakeUnsafeVectorSetLargerSize : std::vector<T> {
+ friend void unsafeVectorSetLargerSizeImpl(std::vector<T>& v, std::size_t n) {
+ // v._M_impl._M_finish += (n - v.size());
+ (v.*Ptr_M_impl).*Ptr_M_finish += (n - v.size());
+ }
+};
+
+#define FOLLY_DECLARE_VECTOR_RESIZE_WITHOUT_INIT(TYPE) \
+ template struct folly::detail::MakeUnsafeVectorSetLargerSize< \
+ FollyMemoryDetailTranslationUnitTag, \
+ TYPE, \
+ std::vector<TYPE>::_Vector_impl( \
+ std::_Vector_base<TYPE, std::allocator<TYPE>>::*), \
+ &std::vector<TYPE>::_M_impl, \
+ TYPE*(std::vector<TYPE>::_Vector_impl::*), \
+ &std::vector<TYPE>::_Vector_impl::_M_finish>; \
+ FOLLY_DECLARE_VECTOR_RESIZE_WITHOUT_INIT_IMPL(TYPE)
+
+#elif defined(_MSC_VER)
+// MSVC
+
+#define FOLLY_DECLARE_VECTOR_RESIZE_WITHOUT_INIT(TYPE) \
+ extern inline void unsafeVectorSetLargerSizeImpl( \
+ std::vector<TYPE>& v, std::size_t n) { \
+ v._Mylast() += (n - v.size()); \
+ } \
+ FOLLY_DECLARE_VECTOR_RESIZE_WITHOUT_INIT_IMPL(TYPE)
+
+#else
+#warn "No implementation for resizeWithoutInitialization of std::vector"
+#endif
+
+} // namespace detail
+} // namespace folly
+
+#if defined(FOLLY_DECLARE_VECTOR_RESIZE_WITHOUT_INIT)
+FOLLY_DECLARE_VECTOR_RESIZE_WITHOUT_INIT(char)
+FOLLY_DECLARE_VECTOR_RESIZE_WITHOUT_INIT(unsigned char)
+#endif
--- /dev/null
+/*
+ * Copyright 2017 Facebook, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "folly/memory/UninitializedMemoryHacks.h"
+
+// Verify that this is okay to put in multiple translation units
+FOLLY_DECLARE_VECTOR_RESIZE_WITHOUT_INIT(int)
--- /dev/null
+/*
+ * Copyright 2017 Facebook, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "folly/memory/UninitializedMemoryHacks.h"
+
+#include <algorithm>
+#include <string>
+#include <vector>
+
+#include <folly/Memory.h>
+#include <folly/Random.h>
+#include <folly/portability/GTest.h>
+#include <glog/logging.h>
+
+void describePlatform() {
+ LOG(INFO) << "sizeof(void*) = " << sizeof(void*);
+
+ LOG(INFO) << "sizeof(std::string) = " << sizeof(std::string);
+#if defined(_LIBCPP_STRING)
+ LOG(INFO) << "std::string from libc++";
+#elif defined(_STLP_STRING)
+ LOG(INFO) << "std::string from STLport";
+#elif defined(_GLIBCXX_USE_FB)
+ LOG(INFO) << "std::string from FBString";
+#elif defined(_GLIBCXX_STRING) && _GLIBCXX_USE_CXX11_ABI
+ LOG(INFO) << "std::string from libstdc++ with SSO";
+#elif defined(_GLIBCXX_STRING)
+ LOG(INFO) << "std::string from old libstdc++";
+#elif defined(_MSC_VER)
+ LOG(INFO) << "std::string from MSVC";
+#else
+ LOG(INFO) << "UNKNOWN std::string implementation";
+#endif
+
+ LOG(INFO) << "sizeof(std::vector<char>) = " << sizeof(std::vector<char>);
+#if defined(_LIBCPP_VECTOR)
+ LOG(INFO) << "std::vector from libc++";
+#elif defined(_STLP_VECTOR)
+ LOG(INFO) << "std::vector from STLport";
+#elif defined(_GLIBCXX_VECTOR)
+ LOG(INFO) << "std::vector from libstdc++";
+#elif defined(_MSC_VER)
+ LOG(INFO) << "std::vector from MSVC";
+#else
+ LOG(INFO) << "UNKNOWN std::vector implementation";
+#endif
+}
+
+// Returns a concatenation of target[i] for those i where valid[i]
+template <typename T>
+T validData(T const& target, std::vector<bool> const& valid) {
+ EXPECT_EQ(target.size(), valid.size());
+ T rv;
+ for (std::size_t i = 0; i < valid.size(); ++i) {
+ if (valid[i]) {
+ rv.push_back(target[i]);
+ }
+ }
+ return rv;
+}
+
+template <typename T>
+void doResizeWithoutInit(
+ T& target,
+ std::vector<bool>& valid,
+ std::size_t newSize) {
+ auto oldSize = target.size();
+ auto before = validData(target, valid);
+ folly::resizeWithoutInitialization(target, newSize);
+ valid.resize(newSize);
+ auto after = validData(target, valid);
+ if (oldSize <= newSize) {
+ EXPECT_EQ(before, after);
+ } else {
+ EXPECT_GE(before.size(), after.size());
+ EXPECT_TRUE(std::equal(after.begin(), after.end(), before.begin()));
+ }
+}
+
+template <typename T>
+void doOverwrite(
+ T& target,
+ std::vector<bool>& valid,
+ std::size_t b,
+ std::size_t e) {
+ for (auto i = b; i < e && i < target.size(); ++i) {
+ target[i] = '0' + (i % 10);
+ valid[i] = true;
+ }
+}
+
+template <typename T>
+void doResize(T& target, std::vector<bool>& valid, std::size_t newSize) {
+ auto oldSize = target.size();
+ auto before = validData(target, valid);
+ target.resize(newSize);
+ valid.resize(newSize);
+ for (auto i = oldSize; i < newSize; ++i) {
+ valid[i] = true;
+ }
+ auto after = validData(target, valid);
+ if (oldSize == newSize) {
+ EXPECT_EQ(before, after);
+ } else if (oldSize < newSize) {
+ EXPECT_LT(before.size(), after.size());
+ EXPECT_TRUE(std::equal(before.begin(), before.end(), after.begin()));
+ } else {
+ EXPECT_GE(before.size(), after.size());
+ EXPECT_TRUE(std::equal(after.begin(), after.end(), before.begin()));
+ }
+}
+
+template <typename T>
+void doClear(T& target, std::vector<bool>& valid) {
+ target.clear();
+ valid.clear();
+}
+
+template <typename T>
+void doInsert(T& target, std::vector<bool>& valid, std::size_t i) {
+ target.insert(target.begin() + i, 'I');
+ valid.insert(valid.begin() + i, true);
+}
+
+template <typename T>
+void doErase(T& target, std::vector<bool>& valid, std::size_t i) {
+ target.erase(target.begin() + i);
+ valid.erase(valid.begin() + i);
+}
+
+template <typename T>
+void doPushBack(T& target, std::vector<bool>& valid) {
+ target.push_back('P');
+ valid.push_back(true);
+}
+
+template <typename T>
+void genericCheck(T& target) {
+ EXPECT_LE(target.size(), target.capacity());
+ EXPECT_EQ(target.size() == 0, target.empty());
+ EXPECT_EQ(target.size(), target.end() - target.begin());
+ EXPECT_EQ(target.size(), target.cend() - target.cbegin());
+ if (!target.empty()) {
+ EXPECT_EQ(target.data(), &target[0]);
+ EXPECT_EQ(target.data(), &target.front());
+ EXPECT_EQ(target.data() + target.size() - 1, &target.back());
+ }
+}
+
+template <typename T>
+void check(T& target) {
+ genericCheck(target);
+}
+
+template <>
+void check<std::string>(std::string& target) {
+ genericCheck(target);
+ EXPECT_EQ(target.c_str(), target.data());
+ EXPECT_EQ(target.c_str()[target.size()], '\0');
+}
+
+template <typename T>
+void testSimple() {
+ describePlatform();
+
+ auto sizes = {0, 1, 10, 14, 15, 16, 17, 22, 23, 24, 32, 95, 100, 10000};
+ for (auto i : sizes) {
+ for (auto j : sizes) {
+ {
+ T target;
+ std::vector<bool> valid;
+ doResize(target, valid, i);
+ doResizeWithoutInit(target, valid, j);
+ check(target);
+ }
+
+ {
+ T target;
+ std::vector<bool> valid;
+ doResize(target, valid, i);
+ doResizeWithoutInit(target, valid, j);
+ doOverwrite(target, valid, i, j);
+ check(target);
+ }
+
+ {
+ T target;
+ std::vector<bool> valid;
+ doResizeWithoutInit(target, valid, i);
+ doResize(target, valid, j);
+ doOverwrite(target, valid, i / 2, i / 2);
+ check(target);
+ }
+
+ {
+ T target;
+ std::vector<bool> valid;
+ doResizeWithoutInit(target, valid, i);
+ doResize(target, valid, j);
+ doOverwrite(target, valid, i, j);
+ check(target);
+ }
+ }
+ }
+}
+
+template <typename T>
+void testRandom(size_t numSteps = 10000) {
+ describePlatform();
+
+ auto target = folly::make_unique<T>();
+ std::vector<bool> valid;
+
+ for (size_t step = 0; step < numSteps; ++step) {
+ auto pct = folly::Random::rand32(100);
+ auto v = folly::Random::rand32(uint32_t{3} << folly::Random::rand32(14));
+
+ if (pct < 5) {
+ doClear(*target, valid);
+ } else if (pct < 30) {
+ T copy;
+ folly::resizeWithoutInitialization(copy, target->size());
+ for (size_t i = 0; i < copy.size(); ++i) {
+ if (valid[i]) {
+ copy[i] = target->at(i);
+ }
+ }
+ if (pct < 10) {
+ std::swap(copy, *target);
+ } else if (pct < 15) {
+ *target = std::move(copy);
+ } else if (pct < 20) {
+ *target = copy;
+ } else if (pct < 25) {
+ target = folly::make_unique<T>(std::move(copy));
+ } else {
+ target = folly::make_unique<T>(copy);
+ }
+ } else if (pct < 35) {
+ target->reserve(v);
+ } else if (pct < 40) {
+ target->shrink_to_fit();
+ } else if (pct < 45) {
+ doResize(*target, valid, v);
+ } else if (pct < 50) {
+ doInsert(*target, valid, v % (target->size() + 1));
+ } else if (pct < 55) {
+ if (!target->empty()) {
+ doErase(*target, valid, v % target->size());
+ }
+ } else if (pct < 60) {
+ doPushBack(*target, valid);
+ } else if (pct < 65) {
+ target = folly::make_unique<T>();
+ valid.clear();
+ } else if (pct < 80) {
+ auto v2 = folly::Random::rand32(uint32_t{3} << folly::Random::rand32(14));
+ doOverwrite(*target, valid, std::min(v, v2), std::max(v, v2));
+ } else {
+ doResizeWithoutInit(*target, valid, v);
+ }
+
+ // don't check every time in implementation does lazy work
+ if (folly::Random::rand32(100) < 50) {
+ check(*target);
+ }
+ }
+}
+
+TEST(UninitializedMemoryHacks, simpleString) {
+ testSimple<std::string>();
+}
+
+TEST(UninitializedMemoryHacks, simpleVectorChar) {
+ testSimple<std::vector<char>>();
+}
+
+TEST(UninitializedMemoryHacks, simpleVectorByte) {
+ testSimple<std::vector<uint8_t>>();
+}
+
+TEST(UninitializedMemoryHacks, simpleVectorInt) {
+ testSimple<std::vector<int>>();
+}
+
+TEST(UninitializedMemoryHacks, randomString) {
+ testRandom<std::string>();
+}
+
+TEST(UninitializedMemoryHacks, randomVectorChar) {
+ testRandom<std::vector<char>>();
+}
+
+TEST(UninitializedMemoryHacks, randomVectorByte) {
+ testRandom<std::vector<uint8_t>>();
+}
+
+TEST(UninitializedMemoryHacks, randomVectorInt) {
+ testRandom<std::vector<int>>();
+}
+
+// We are deliberately putting this at the bottom to make sure it can follow use
+FOLLY_DECLARE_VECTOR_RESIZE_WITHOUT_INIT(int)
projects / folly.git / commitdiff