From: James Sedgwick Date: Sat, 21 Oct 2017 20:00:55 +0000 (-0700) Subject: move MemoryMapping, Shell, ThreadId, ThreadName, and VersionCheck to system/ X-Git-Tag: v2017.10.23.00~2 X-Git-Url: http://demsky.eecs.uci.edu/git/?a=commitdiff_plain;h=87a2402c6ba1c8610b034ed39a896fcc3b61deec;p=folly.git move MemoryMapping, Shell, ThreadId, ThreadName, and VersionCheck to system/ Summary: Everything that's going in system/ besides CpuId and Subprocess, which are included in hphp Reviewed By: mzlee Differential Revision: D6102263 fbshipit-source-id: 564ef584c341a4ac79db14a9d58fe23ce51e78b3 --- diff --git a/CMakeLists.txt b/CMakeLists.txt index 3255fe47..6d9237e9 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -480,6 +480,13 @@ if (BUILD_TESTS) DIRECTORY synchronization/test/ TEST call_once_test SOURCES CallOnceTest.cpp + DIRECTORY system/test/ + TEST memory_mapping_test SOURCES MemoryMappingTest.cpp + TEST shell_test SOURCES ShellTest.cpp + #TEST subprocess_test SOURCES SubprocessTest.cpp + TEST thread_id_test SOURCES ThreadIdTest.cpp + TEST thread_name_test SOURCES ThreadNameTest.cpp + DIRECTORY test/ TEST ahm_int_stress_test SOURCES AHMIntStressTest.cpp TEST arena_test SOURCES ArenaTest.cpp @@ -552,7 +559,6 @@ if (BUILD_TESTS) TEST map_util_test SOURCES MapUtilTest.cpp TEST memcpy_test SOURCES MemcpyTest.cpp TEST memory_idler_test SOURCES MemoryIdlerTest.cpp - TEST memory_mapping_test SOURCES MemoryMappingTest.cpp TEST memory_test SOURCES MemoryTest.cpp TEST merge SOURCES MergeTest.cpp TEST move_wrapper_test SOURCES MoveWrapperTest.cpp @@ -579,7 +585,6 @@ if (BUILD_TESTS) TEST scope_guard_test SOURCES ScopeGuardTest.cpp # Heavily dependent on drand and srand48 #TEST shared_mutex_test SOURCES SharedMutexTest.cpp - TEST shell_test SOURCES ShellTest.cpp TEST singleton_test SOURCES SingletonTest.cpp TEST singleton_test_global SOURCES SingletonTestGlobal.cpp TEST singleton_thread_local_test SOURCES SingletonThreadLocalTest.cpp @@ -588,13 +593,10 @@ if (BUILD_TESTS) TEST sorted_vector_types_test SOURCES sorted_vector_test.cpp TEST sparse_byte_set_test SOURCES SparseByteSetTest.cpp TEST string_test SOURCES StringTest.cpp - #TEST subprocess_test SOURCES SubprocessTest.cpp TEST synchronized_test SOURCES SynchronizedTest.cpp TEST thread_cached_arena_test SOURCES ThreadCachedArenaTest.cpp TEST thread_cached_int_test SOURCES ThreadCachedIntTest.cpp - TEST thread_id_test SOURCES ThreadIdTest.cpp TEST thread_local_test SOURCES ThreadLocalTest.cpp - TEST thread_name_test SOURCES ThreadNameTest.cpp TEST timeout_queue_test SOURCES TimeoutQueueTest.cpp TEST token_bucket_test SOURCES TokenBucketTest.cpp TEST traits_test SOURCES TraitsTest.cpp diff --git a/folly/Makefile.am b/folly/Makefile.am index ef127982..cbc950b2 100644 --- a/folly/Makefile.am +++ b/folly/Makefile.am @@ -327,7 +327,6 @@ nobase_follyinclude_HEADERS = \ MapUtil.h \ Math.h \ Memory.h \ - MemoryMapping.h \ memory/MallctlHelper.h \ memory/UninitializedMemoryHacks.h \ MicroSpinLock.h \ @@ -390,7 +389,6 @@ nobase_follyinclude_HEADERS = \ SafeAssert.h \ ScopeGuard.h \ SharedMutex.h \ - Shell.h \ Singleton.h \ Singleton-inl.h \ SingletonThreadLocal.h \ @@ -419,6 +417,11 @@ nobase_follyinclude_HEADERS = \ stats/TimeseriesHistogram-defs.h \ stats/TimeseriesHistogram.h \ synchronization/CallOnce.h \ + system/MemoryMapping.h \ + system/Shell.h \ + system/ThreadId.h \ + system/ThreadName.h \ + system/VersionCheck.h \ stop_watch.h \ String.h \ String-inl.h \ @@ -434,9 +437,7 @@ nobase_follyinclude_HEADERS = \ test/TestUtils.h \ ThreadCachedArena.h \ ThreadCachedInt.h \ - ThreadId.h \ ThreadLocal.h \ - ThreadName.h \ TimeoutQueue.h \ TokenBucket.h \ tracing/StaticTracepoint.h \ @@ -452,8 +453,7 @@ nobase_follyinclude_HEADERS = \ Uri.h \ Uri-inl.h \ Utility.h \ - Varint.h \ - VersionCheck.h + Varint.h FormatTables.cpp: build/generate_format_tables.py $(PYTHON) build/generate_format_tables.py @@ -563,7 +563,6 @@ libfolly_la_SOURCES = \ detail/MemoryIdler.cpp \ detail/SocketFastOpen.cpp \ MacAddress.cpp \ - MemoryMapping.cpp \ portability/Dirent.cpp \ portability/Fcntl.cpp \ portability/Libgen.cpp \ @@ -588,7 +587,6 @@ libfolly_la_SOURCES = \ SafeAssert.cpp \ ScopeGuard.cpp \ SharedMutex.cpp \ - Shell.cpp \ MicroLock.cpp \ Optional.cpp \ Singleton.cpp \ @@ -602,13 +600,15 @@ libfolly_la_SOURCES = \ stats/Histogram.cpp \ stats/MultiLevelTimeSeries.cpp \ stats/TimeseriesHistogram.cpp \ + system/MemoryMapping.cpp \ + system/Shell.cpp \ + system/ThreadName.cpp \ + system/VersionCheck.cpp \ Subprocess.cpp \ ThreadCachedArena.cpp \ - ThreadName.cpp \ TimeoutQueue.cpp \ Try.cpp \ Uri.cpp \ - Version.cpp \ experimental/AsymmetricMemoryBarrier.cpp \ experimental/ThreadedRepeatingFunctionRunner.cpp \ experimental/bser/Dump.cpp \ diff --git a/folly/MemoryMapping.cpp b/folly/MemoryMapping.cpp deleted file mode 100644 index 30d6a1fe..00000000 --- a/folly/MemoryMapping.cpp +++ /dev/null @@ -1,407 +0,0 @@ -/* - * Copyright 2017 Facebook, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include - -#include -#include -#include - -#include -#include -#include - -#ifdef __linux__ -#include -#endif - -#include -#include -#include - -static constexpr ssize_t kDefaultMlockChunkSize = -#ifndef _MSC_VER - // Linux implementations of unmap/mlock/munlock take a kernel - // semaphore and block other threads from doing other memory - // operations. Split the operations in chunks. - (1 << 20) // 1MB -#else // _MSC_VER - // MSVC doesn't have this problem, and calling munmap many times - // with the same address is a bad idea with the windows implementation. - (-1) -#endif // _MSC_VER - ; - -DEFINE_int64(mlock_chunk_size, kDefaultMlockChunkSize, - "Maximum bytes to mlock/munlock/munmap at once " - "(will be rounded up to PAGESIZE). Ignored if negative."); - -#ifndef MAP_POPULATE -#define MAP_POPULATE 0 -#endif - -namespace folly { - -MemoryMapping::MemoryMapping(MemoryMapping&& other) noexcept { - swap(other); -} - -MemoryMapping::MemoryMapping(File file, off_t offset, off_t length, - Options options) - : file_(std::move(file)), - options_(std::move(options)) { - CHECK(file_); - init(offset, length); -} - -MemoryMapping::MemoryMapping(const char* name, off_t offset, off_t length, - Options options) - : MemoryMapping(File(name, options.writable ? O_RDWR : O_RDONLY), - offset, - length, - options) { } - -MemoryMapping::MemoryMapping(int fd, off_t offset, off_t length, - Options options) - : MemoryMapping(File(fd), offset, length, options) { } - -MemoryMapping::MemoryMapping(AnonymousType, off_t length, Options options) - : options_(std::move(options)) { - init(0, length); -} - -namespace { - -#ifdef __linux__ -void getDeviceOptions(dev_t device, off_t& pageSize, bool& autoExtend) { - auto ps = getHugePageSizeForDevice(device); - if (ps) { - pageSize = ps->size; - autoExtend = true; - } -} -#else -inline void getDeviceOptions(dev_t, off_t&, bool&) {} -#endif - -} // namespace - -void MemoryMapping::init(off_t offset, off_t length) { - const bool grow = options_.grow; - const bool anon = !file_; - CHECK(!(grow && anon)); - - off_t& pageSize = options_.pageSize; - - struct stat st; - - // On Linux, hugetlbfs file systems don't require ftruncate() to grow the - // file, and (on kernels before 2.6.24) don't even allow it. Also, the file - // size is always a multiple of the page size. - bool autoExtend = false; - - if (!anon) { - // Stat the file - CHECK_ERR(fstat(file_.fd(), &st)); - - if (pageSize == 0) { - getDeviceOptions(st.st_dev, pageSize, autoExtend); - } - } else { - DCHECK(!file_); - DCHECK_EQ(offset, 0); - CHECK_EQ(pageSize, 0); - CHECK_GE(length, 0); - } - - if (pageSize == 0) { - pageSize = off_t(sysconf(_SC_PAGESIZE)); - } - - CHECK_GT(pageSize, 0); - CHECK_EQ(pageSize & (pageSize - 1), 0); // power of two - CHECK_GE(offset, 0); - - // Round down the start of the mapped region - off_t skipStart = offset % pageSize; - offset -= skipStart; - - mapLength_ = length; - if (mapLength_ != -1) { - mapLength_ += skipStart; - - // Round up the end of the mapped region - mapLength_ = (mapLength_ + pageSize - 1) / pageSize * pageSize; - } - - off_t remaining = anon ? length : st.st_size - offset; - - if (mapLength_ == -1) { - length = mapLength_ = remaining; - } else { - if (length > remaining) { - if (grow) { - if (!autoExtend) { - PCHECK(0 == ftruncate(file_.fd(), offset + length)) - << "ftruncate() failed, couldn't grow file to " - << offset + length; - remaining = length; - } else { - // Extend mapping to multiple of page size, don't use ftruncate - remaining = mapLength_; - } - } else { - length = remaining; - } - } - if (mapLength_ > remaining) { - mapLength_ = remaining; - } - } - - if (length == 0) { - mapLength_ = 0; - mapStart_ = nullptr; - } else { - int flags = options_.shared ? MAP_SHARED : MAP_PRIVATE; - if (anon) { - flags |= MAP_ANONYMOUS; - } - if (options_.prefault) { - flags |= MAP_POPULATE; - } - - // The standard doesn't actually require PROT_NONE to be zero... - int prot = PROT_NONE; - if (options_.readable || options_.writable) { - prot = ((options_.readable ? PROT_READ : 0) | - (options_.writable ? PROT_WRITE : 0)); - } - - unsigned char* start = static_cast(mmap( - options_.address, size_t(mapLength_), prot, flags, file_.fd(), offset)); - PCHECK(start != MAP_FAILED) - << " offset=" << offset - << " length=" << mapLength_; - mapStart_ = start; - data_.reset(start + skipStart, size_t(length)); - } -} - -namespace { - -off_t memOpChunkSize(off_t length, off_t pageSize) { - off_t chunkSize = length; - if (FLAGS_mlock_chunk_size <= 0) { - return chunkSize; - } - - chunkSize = off_t(FLAGS_mlock_chunk_size); - off_t r = chunkSize % pageSize; - if (r) { - chunkSize += (pageSize - r); - } - return chunkSize; -} - -/** - * Run @op in chunks over the buffer @mem of @bufSize length. - * - * Return: - * - success: true + amountSucceeded == bufSize (op success on whole buffer) - * - failure: false + amountSucceeded == nr bytes on which op succeeded. - */ -bool memOpInChunks(std::function op, - void* mem, size_t bufSize, off_t pageSize, - size_t& amountSucceeded) { - // Linux' unmap/mlock/munlock take a kernel semaphore and block other threads - // from doing other memory operations. If the size of the buffer is big the - // semaphore can be down for seconds (for benchmarks see - // http://kostja-osipov.livejournal.com/42963.html). Doing the operations in - // chunks breaks the locking into intervals and lets other threads do memory - // operations of their own. - - size_t chunkSize = size_t(memOpChunkSize(off_t(bufSize), pageSize)); - - char* addr = static_cast(mem); - amountSucceeded = 0; - - while (amountSucceeded < bufSize) { - size_t size = std::min(chunkSize, bufSize - amountSucceeded); - if (op(addr + amountSucceeded, size) != 0) { - return false; - } - amountSucceeded += size; - } - - return true; -} - -} // namespace - -bool MemoryMapping::mlock(LockMode lock) { - size_t amountSucceeded = 0; - locked_ = memOpInChunks( - ::mlock, - mapStart_, - size_t(mapLength_), - options_.pageSize, - amountSucceeded); - if (locked_) { - return true; - } - - auto msg = - folly::format("mlock({}) failed at {}", mapLength_, amountSucceeded); - if (lock == LockMode::TRY_LOCK && errno == EPERM) { - PLOG(WARNING) << msg; - } else if (lock == LockMode::TRY_LOCK && errno == ENOMEM) { - VLOG(1) << msg; - } else { - PLOG(FATAL) << msg; - } - - // only part of the buffer was mlocked, unlock it back - if (!memOpInChunks(::munlock, mapStart_, amountSucceeded, options_.pageSize, - amountSucceeded)) { - PLOG(WARNING) << "munlock()"; - } - - return false; -} - -void MemoryMapping::munlock(bool dontneed) { - if (!locked_) { - return; - } - - size_t amountSucceeded = 0; - if (!memOpInChunks( - ::munlock, - mapStart_, - size_t(mapLength_), - options_.pageSize, - amountSucceeded)) { - PLOG(WARNING) << "munlock()"; - } - if (mapLength_ && dontneed && - ::madvise(mapStart_, size_t(mapLength_), MADV_DONTNEED)) { - PLOG(WARNING) << "madvise()"; - } - locked_ = false; -} - -void MemoryMapping::hintLinearScan() { - advise(MADV_SEQUENTIAL); -} - -MemoryMapping::~MemoryMapping() { - if (mapLength_) { - size_t amountSucceeded = 0; - if (!memOpInChunks( - ::munmap, - mapStart_, - size_t(mapLength_), - options_.pageSize, - amountSucceeded)) { - PLOG(FATAL) << folly::format("munmap({}) failed at {}", - mapLength_, amountSucceeded); - } - } -} - -void MemoryMapping::advise(int advice) const { - advise(advice, 0, size_t(mapLength_)); -} - -void MemoryMapping::advise(int advice, size_t offset, size_t length) const { - CHECK_LE(offset + length, size_t(mapLength_)) - << " offset: " << offset - << " length: " << length - << " mapLength_: " << mapLength_; - - // Include the entire start page: round down to page boundary. - const auto offMisalign = offset % options_.pageSize; - offset -= offMisalign; - length += offMisalign; - - // Round the last page down to page boundary. - if (offset + length != size_t(mapLength_)) { - length -= length % options_.pageSize; - } - - if (length == 0) { - return; - } - - char* mapStart = static_cast(mapStart_) + offset; - PLOG_IF(WARNING, ::madvise(mapStart, length, advice)) << "madvise"; -} - -MemoryMapping& MemoryMapping::operator=(MemoryMapping other) { - swap(other); - return *this; -} - -void MemoryMapping::swap(MemoryMapping& other) noexcept { - using std::swap; - swap(this->file_, other.file_); - swap(this->mapStart_, other.mapStart_); - swap(this->mapLength_, other.mapLength_); - swap(this->options_, other.options_); - swap(this->locked_, other.locked_); - swap(this->data_, other.data_); -} - -void swap(MemoryMapping& a, MemoryMapping& b) noexcept { a.swap(b); } - -void alignedForwardMemcpy(void* dst, const void* src, size_t size) { - assert(reinterpret_cast(src) % alignof(unsigned long) == 0); - assert(reinterpret_cast(dst) % alignof(unsigned long) == 0); - - auto srcl = static_cast(src); - auto dstl = static_cast(dst); - - while (size >= sizeof(unsigned long)) { - *dstl++ = *srcl++; - size -= sizeof(unsigned long); - } - - auto srcc = reinterpret_cast(srcl); - auto dstc = reinterpret_cast(dstl); - - while (size != 0) { - *dstc++ = *srcc++; - --size; - } -} - -void mmapFileCopy(const char* src, const char* dest, mode_t mode) { - MemoryMapping srcMap(src); - srcMap.hintLinearScan(); - - MemoryMapping destMap( - File(dest, O_RDWR | O_CREAT | O_TRUNC, mode), - 0, - off_t(srcMap.range().size()), - MemoryMapping::writable()); - - alignedForwardMemcpy(destMap.writableRange().data(), - srcMap.range().data(), - srcMap.range().size()); -} - -} // namespace folly diff --git a/folly/MemoryMapping.h b/folly/MemoryMapping.h deleted file mode 100644 index 38220632..00000000 --- a/folly/MemoryMapping.h +++ /dev/null @@ -1,251 +0,0 @@ -/* - * Copyright 2017 Facebook, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include -#include - -#include -#include - -namespace folly { - -/** - * Maps files in memory (read-only). - * - * @author Tudor Bosman (tudorb@fb.com) - */ -class MemoryMapping : boost::noncopyable { - public: - /** - * Lock the pages in memory? - * TRY_LOCK = try to lock, log warning if permission denied - * MUST_LOCK = lock, fail assertion if permission denied. - */ - enum class LockMode { - TRY_LOCK, - MUST_LOCK - }; - /** - * Map a portion of the file indicated by filename in memory, causing a CHECK - * failure on error. - * - * By default, map the whole file. length=-1: map from offset to EOF. - * Unlike the mmap() system call, offset and length don't need to be - * page-aligned. length is clipped to the end of the file if it's too large. - * - * The mapping will be destroyed (and the memory pointed-to by data() will - * likely become inaccessible) when the MemoryMapping object is destroyed. - */ - struct Options { - Options() {} - - // Convenience methods; return *this for chaining. - Options& setPageSize(off_t v) { pageSize = v; return *this; } - Options& setShared(bool v) { shared = v; return *this; } - Options& setPrefault(bool v) { prefault = v; return *this; } - Options& setReadable(bool v) { readable = v; return *this; } - Options& setWritable(bool v) { writable = v; return *this; } - Options& setGrow(bool v) { grow = v; return *this; } - - // Page size. 0 = use appropriate page size. - // (On Linux, we use a huge page size if the file is on a hugetlbfs - // file system, and the default page size otherwise) - off_t pageSize = 0; - - // If shared (default), the memory mapping is shared with other processes - // mapping the same file (or children); if not shared (private), each - // process has its own mapping. Changes in writable, private mappings are - // not reflected to the underlying file. See the discussion of - // MAP_PRIVATE vs MAP_SHARED in the mmap(2) manual page. - bool shared = true; - - // Populate page tables; subsequent accesses should not be blocked - // by page faults. This is a hint, as it may not be supported. - bool prefault = false; - - // Map the pages readable. Note that mapping pages without read permissions - // is not universally supported (not supported on hugetlbfs on Linux, for - // example) - bool readable = true; - - // Map the pages writable. - bool writable = false; - - // When mapping a file in writable mode, grow the file to the requested - // length (using ftruncate()) before mapping; if false, truncate the - // mapping to the actual file size instead. - bool grow = false; - - // Fix map at this address, if not nullptr. Must be aligned to a multiple - // of the appropriate page size. - void* address = nullptr; - }; - - // Options to emulate the old WritableMemoryMapping: readable and writable, - // allow growing the file if mapping past EOF. - static Options writable() { - return Options().setWritable(true).setGrow(true); - } - - enum AnonymousType { - kAnonymous - }; - - /** - * Create an anonymous mapping. - */ - MemoryMapping(AnonymousType, off_t length, Options options=Options()); - - explicit MemoryMapping(File file, - off_t offset=0, - off_t length=-1, - Options options=Options()); - - explicit MemoryMapping(const char* name, - off_t offset=0, - off_t length=-1, - Options options=Options()); - - explicit MemoryMapping(int fd, - off_t offset=0, - off_t length=-1, - Options options=Options()); - - MemoryMapping(MemoryMapping&&) noexcept; - - ~MemoryMapping(); - - MemoryMapping& operator=(MemoryMapping); - - void swap(MemoryMapping& other) noexcept; - - /** - * Lock the pages in memory - */ - bool mlock(LockMode lock); - - /** - * Unlock the pages. - * If dontneed is true, the kernel is instructed to release these pages - * (per madvise(MADV_DONTNEED)). - */ - void munlock(bool dontneed = false); - - /** - * Hint that these pages will be scanned linearly. - * madvise(MADV_SEQUENTIAL) - */ - void hintLinearScan(); - - /** - * Advise the kernel about memory access. - */ - void advise(int advice) const; - void advise(int advice, size_t offset, size_t length) const; - - /** - * A bitwise cast of the mapped bytes as range of values. Only intended for - * use with POD or in-place usable types. - */ - template - Range asRange() const { - size_t count = data_.size() / sizeof(T); - return Range(static_cast( - static_cast(data_.data())), - count); - } - - /** - * A range of bytes mapped by this mapping. - */ - ByteRange range() const { - return data_; - } - - /** - * A bitwise cast of the mapped bytes as range of mutable values. Only - * intended for use with POD or in-place usable types. - */ - template - Range asWritableRange() const { - DCHECK(options_.writable); // you'll segfault anyway... - size_t count = data_.size() / sizeof(T); - return Range(static_cast( - static_cast(data_.data())), - count); - } - - /** - * A range of mutable bytes mapped by this mapping. - */ - MutableByteRange writableRange() const { - DCHECK(options_.writable); // you'll segfault anyway... - return data_; - } - - /** - * Return the memory area where the file was mapped. - * Deprecated; use range() instead. - */ - StringPiece data() const { - return asRange(); - } - - bool mlocked() const { - return locked_; - } - - int fd() const { return file_.fd(); } - - private: - MemoryMapping(); - - enum InitFlags { - kGrow = 1 << 0, - kAnon = 1 << 1, - }; - void init(off_t offset, off_t length); - - File file_; - void* mapStart_ = nullptr; - off_t mapLength_ = 0; - Options options_; - bool locked_ = false; - MutableByteRange data_; -}; - -void swap(MemoryMapping&, MemoryMapping&) noexcept; - -/** - * A special case of memcpy() that always copies memory forwards. - * (libc's memcpy() is allowed to copy memory backwards, and will do so - * when using SSSE3 instructions). - * - * Assumes src and dest are aligned to alignof(unsigned long). - * - * Useful when copying from/to memory mappings after hintLinearScan(); - * copying backwards renders any prefetching useless (even harmful). - */ -void alignedForwardMemcpy(void* dest, const void* src, size_t size); - -/** - * Copy a file using mmap(). Overwrites dest. - */ -void mmapFileCopy(const char* src, const char* dest, mode_t mode = 0666); - -} // namespace folly diff --git a/folly/Shell.cpp b/folly/Shell.cpp deleted file mode 100644 index 3ce31b76..00000000 --- a/folly/Shell.cpp +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright 2017 Facebook, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include - -namespace folly { - -std::string shellQuote(StringPiece argument) { - std::string quoted = "'"; - for (auto c : argument) { - if (c == '\'') { - quoted += "'\\''"; - } else { - quoted += c; - } - } - return quoted + "'"; -} - -} // namespace folly diff --git a/folly/Shell.h b/folly/Shell.h deleted file mode 100644 index cefe2ddb..00000000 --- a/folly/Shell.h +++ /dev/null @@ -1,93 +0,0 @@ -/* - * Copyright 2017 Facebook, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * `Shell` provides a collection of functions to use with `Subprocess` that make - * it easier to safely run processes in a unix shell. - * - * Note: use this rarely and carefully. By default you should use `Subprocess` - * with a vector of arguments. - */ - -#pragma once - -#include -#include - -#include -#include -#include - -namespace folly { - -/** - * Quotes an argument to make it suitable for use as shell command arguments. - */ -std::string shellQuote(StringPiece argument); - -namespace detail { -template -std::vector shellify( - StringPiece format, - Arguments&&... arguments) { - auto command = sformat( - format, - shellQuote(to(std::forward(arguments)))...); - return {"/bin/sh", "-c", command}; -} - -struct ShellCmdFormat { - StringPiece format; - template - std::vector operator()(Arguments&&... arguments) const { - return ::folly::detail::shellify( - format, std::forward(arguments)...); - } -}; - -} // namespace detail - -inline namespace literals { -inline namespace shell_literals { -constexpr detail::ShellCmdFormat operator"" _shellify( - char const* name, - std::size_t length) { - return {folly::StringPiece(name, length)}; -} -} // inline namespace shell_literals -} // inline namespace literals - -/** - * Create argument array for `Subprocess()` for a process running in a - * shell. - * - * The shell to use is always going to be `/bin/sh`. - * - * This is deprecated in favour of the user-defined-literal `_shellify` - * from namespace `folly::shell_literals` because that requires that the format - * string is a compile-time constant which can be inspected during code reviews - */ -template -FOLLY_DEPRECATED( - "Use `\"command {} {} ...\"_shellify(argument1, argument2 ...)` from " - "namespace `folly::literals::shell_literals`") -std::vector shellify( - StringPiece format, - Arguments&&... arguments) { - return detail::shellify(format, std::forward(arguments)...); -} - -} // namespace folly diff --git a/folly/Subprocess.cpp b/folly/Subprocess.cpp index 7ac96687..77706213 100644 --- a/folly/Subprocess.cpp +++ b/folly/Subprocess.cpp @@ -38,13 +38,13 @@ #include #include #include -#include #include #include #include #include #include #include +#include constexpr int kExecFailure = 127; constexpr int kChildFailure = 126; diff --git a/folly/ThreadId.h b/folly/ThreadId.h deleted file mode 100644 index 18b82cd7..00000000 --- a/folly/ThreadId.h +++ /dev/null @@ -1,91 +0,0 @@ -/* - * Copyright 2017 Facebook, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include - -#include -#include -#include -#include - -namespace folly { - -/** - * Get a process-specific identifier for the current thread. - * - * The return value will uniquely identify the thread within the current - * process. - * - * Note that the return value does not necessarily correspond to an operating - * system thread ID. The return value is also only unique within the current - * process: getCurrentThreadID() may return the same value for two concurrently - * running threads in separate processes. - * - * The thread ID may be reused once the thread it corresponds to has been - * joined. - */ -inline uint64_t getCurrentThreadID() { -#if __APPLE__ - return uint64_t(pthread_mach_thread_np(pthread_self())); -#elif _WIN32 - return uint64_t(GetCurrentThreadId()); -#else - return uint64_t(pthread_self()); -#endif -} - -/** - * Get the operating-system level thread ID for the current thread. - * - * The returned value will uniquely identify this thread on the system. - * - * This makes it more suitable for logging or displaying in user interfaces - * than the result of getCurrentThreadID(). - * - * There are some potential caveats about this API, however: - * - * - In theory there is no guarantee that application threads map one-to-one to - * kernel threads. An application threading implementation could potentially - * share one OS thread across multiple application threads, and/or it could - * potentially move application threads between different OS threads over - * time. However, in practice all of the platforms we currently support have - * a one-to-one mapping between userspace threads and operating system - * threads. - * - * - This API may also be slightly slower than getCurrentThreadID() on some - * platforms. This API may require a system call, where getCurrentThreadID() - * may only need to read thread-local memory. - * - * On Linux the returned value is a pid_t, and can be used in contexts - * requiring a thread pid_t. - * - * The thread ID may be reused once the thread it corresponds to has been - * joined. - */ -inline uint64_t getOSThreadID() { -#if __APPLE__ - uint64_t tid; - pthread_threadid_np(nullptr, &tid); - return tid; -#elif _WIN32 - return uint64_t(GetCurrentThreadId()); -#else - return uint64_t(syscall(FOLLY_SYS_gettid)); -#endif -} -} diff --git a/folly/ThreadName.cpp b/folly/ThreadName.cpp deleted file mode 100644 index abc7eaad..00000000 --- a/folly/ThreadName.cpp +++ /dev/null @@ -1,151 +0,0 @@ -/* - * Copyright 2017 Facebook, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include - -#include - -#include -#include -#include - -namespace folly { - -// This looks a bit weird, but it's necessary to avoid -// having an undefined compiler function called. -#if defined(__GLIBC__) && !defined(__APPLE__) && !defined(__ANDROID__) -#if __GLIBC_PREREQ(2, 12) -// has pthread_setname_np(pthread_t, const char*) (2 params) -#define FOLLY_HAS_PTHREAD_SETNAME_NP_THREAD_NAME 1 -#endif -#endif - -#if defined(__APPLE__) && defined(__MAC_OS_X_VERSION_MIN_REQUIRED) -#if __MAC_OS_X_VERSION_MIN_REQUIRED >= 1060 -// has pthread_setname_np(const char*) (1 param) -#define FOLLY_HAS_PTHREAD_SETNAME_NP_NAME 1 -#endif -#endif - -namespace { - -#if FOLLY_HAVE_PTHREAD && !_WIN32 -pthread_t stdTidToPthreadId(std::thread::id tid) { - static_assert( - std::is_same::value, - "This assumes that the native handle type is pthread_t"); - static_assert( - sizeof(std::thread::native_handle_type) == sizeof(std::thread::id), - "This assumes std::thread::id is a thin wrapper around " - "std::thread::native_handle_type, but that doesn't appear to be true."); - // In most implementations, std::thread::id is a thin wrapper around - // std::thread::native_handle_type, which means we can do unsafe things to - // extract it. - pthread_t id; - std::memcpy(&id, &tid, sizeof(id)); - return id; -} -#endif - -} // namespace - -bool canSetCurrentThreadName() { -#if FOLLY_HAS_PTHREAD_SETNAME_NP_THREAD_NAME || \ - FOLLY_HAS_PTHREAD_SETNAME_NP_NAME - return true; -#else - return false; -#endif -} - -bool canSetOtherThreadName() { -#if FOLLY_HAS_PTHREAD_SETNAME_NP_THREAD_NAME - return true; -#else - return false; -#endif -} - -static constexpr size_t kMaxThreadNameLength = 16; - -Optional getThreadName(std::thread::id id) { -#if FOLLY_HAS_PTHREAD_SETNAME_NP_THREAD_NAME || \ - FOLLY_HAS_PTHREAD_SETNAME_NP_NAME - std::array buf; - if (pthread_getname_np(stdTidToPthreadId(id), buf.data(), buf.size()) != 0) { - return Optional(); - } - return make_optional(std::string(buf.data())); -#else - return Optional(); -#endif -} - -Optional getCurrentThreadName() { - return getThreadName(std::this_thread::get_id()); -} - -bool setThreadName(std::thread::id tid, StringPiece name) { -#if !FOLLY_HAVE_PTHREAD || _WIN32 - return false; -#else - name = name.subpiece(0, kMaxThreadNameLength - 1); - char buf[kMaxThreadNameLength] = {}; - std::memcpy(buf, name.data(), name.size()); - auto id = stdTidToPthreadId(tid); -#if FOLLY_HAS_PTHREAD_SETNAME_NP_THREAD_NAME - return 0 == pthread_setname_np(id, buf); -#elif FOLLY_HAS_PTHREAD_SETNAME_NP_NAME - // Since OS X 10.6 it is possible for a thread to set its own name, - // but not that of some other thread. - if (pthread_equal(pthread_self(), id)) { - return 0 == pthread_setname_np(buf); - } - return false; -#else - (void)id; - return false; -#endif -#endif -} - -#if FOLLY_HAVE_PTHREAD -bool setThreadName(pthread_t pid, StringPiece name) { -#if _WIN32 - // Not currently supported on Windows. - return false; -#else - static_assert( - std::is_same::value, - "This assumes that the native handle type is pthread_t"); - static_assert( - sizeof(std::thread::native_handle_type) == sizeof(std::thread::id), - "This assumes std::thread::id is a thin wrapper around " - "std::thread::native_handle_type, but that doesn't appear to be true."); - // In most implementations, std::thread::id is a thin wrapper around - // std::thread::native_handle_type, which means we can do unsafe things to - // extract it. - std::thread::id id; - std::memcpy(&id, &pid, sizeof(id)); - return setThreadName(id, name); -#endif -} -#endif - -bool setThreadName(StringPiece name) { - return setThreadName(std::this_thread::get_id(), name); -} -} diff --git a/folly/ThreadName.h b/folly/ThreadName.h deleted file mode 100644 index ba7bde25..00000000 --- a/folly/ThreadName.h +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Copyright 2017 Facebook, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include -#include - -#include -#include -#include -#include - -namespace folly { - -/** - * This returns true if the current platform supports setting the name of the - * current thread. - */ -bool canSetCurrentThreadName(); - -/** - * This returns true if the current platform supports setting the name of - * threads other than the one currently executing. - */ -bool canSetOtherThreadName(); - -/** - * Get the name of the given thread, or nothing if an error occurs - * or the functionality is not available. - */ -Optional getThreadName(std::thread::id tid); - -/** - * Equivalent to getThreadName(std::this_thread::get_id()); - */ -Optional getCurrentThreadName(); - -/** - * Set the name of the given thread. - * Returns false on failure, if an error occurs or the functionality - * is not available. - */ -bool setThreadName(std::thread::id tid, StringPiece name); -#if FOLLY_HAVE_PTHREAD -bool setThreadName(pthread_t pid, StringPiece name); -#endif - -/** - * Equivalent to setThreadName(std::this_thread::get_id(), name); - */ -bool setThreadName(StringPiece name); -} diff --git a/folly/Version.cpp b/folly/Version.cpp deleted file mode 100644 index cdacce51..00000000 --- a/folly/Version.cpp +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Copyright 2017 Facebook, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include - -namespace folly { namespace detail { - -FOLLY_VERSION_CHECK(folly, FOLLY_VERSION) - -}} // namespaces diff --git a/folly/VersionCheck.h b/folly/VersionCheck.h deleted file mode 100644 index 049556ef..00000000 --- a/folly/VersionCheck.h +++ /dev/null @@ -1,108 +0,0 @@ -/* - * Copyright 2017 Facebook, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include -#include -#include - -#include -#include - -/** - * Check if the currently loaded version of a library is what you expect. - * - * It is possible for multiple versions of the same shared library to end up - * being loaded simultaneously in the same address space, usually with - * disastrous results. - * - * For example, let's say you have a shared library (foo) that doesn't keep - * binary compatbility between releases, and so each version is distributed as - * a SO with different SONAME. Let's say you build another shared library, bar - * that depends on version 1 of foo: libbar.so depends on libfoo1.so. - * Your main executable now (baz) depends on version 2 of foo, and also - * depends on bar: baz depends on libfoo2.so and libbar.so. - * - * At load time, baz loads libfoo2.so first, then libbar.so; libbar.so will - * load libfoo1.so, but, as this is normal dynamic loading (and not explicit - * dlopen calls with RTLD_DEEPBIND), any symbols from libfoo1.so that are - * also present in libfoo2.so will be satisfied from the (already loaded) - * libfoo2.so. - * - * But foo does not preserve binary compatibility between versions, so all - * hell breaks loose (the symbols from libfoo2.so are not necessarily direct - * replacements of the identically-named symbols in libfoo1.so). - * - * It is better to crash with a helpful error message instead, which is what - * this macro provides. FOLLY_VERSION_CHECK verifies at load time that - * the compiled-in version is the same as the currently loaded version. - * - * Usage: use this macro at namespace scope in a .cpp file (IMPORTANT: NOT - * in the unnamed namespace): - * - * FOLLY_VERSION_CHECK(mylib, "1") - * - * The first argument identifies your library; the second argument is a - * string literal containing the desired version string. - * - * In order to avoid changing the file for each version, the version string - * could be provided on the compiler command line with -D: - * - * FOLLY_VERSION_CHECK(mylib, MYLIB_VERSION) - * - * ... and then commpile your file with -DMYLIB_VERSION=\"1\" - */ - -#if defined(_MSC_VER) -// MSVC doesn't support constructor priorities. Just pray it works, I guess. -// We could implement a link-time mechanism for MSVC, -// via #pragma detect_mismatch but that would only handle -// static library linking. -# define FOLLY_VERSION_CHECK_PRIORITY(Ret, name) \ - __pragma(section(".CRT$XCU",read)) \ - static Ret __cdecl name(void); \ - __declspec(allocate(".CRT$XCU")) \ - Ret (__cdecl*name##_)(void) = name; \ - Ret __cdecl name() - -#elif defined(__APPLE__) -// OS X doesn't support constructor priorities. Just pray it works, I guess. -# define FOLLY_VERSION_CHECK_PRIORITY(Ret, name) \ - __attribute__((__constructor__)) Ret name() - -#else -# define FOLLY_VERSION_CHECK_PRIORITY(Ret, name) \ - __attribute__((__constructor__(101))) Ret name() -#endif - -// Note that this is carefully crafted: PRODUCT##Version must have external -// linkage (so it collides among versions), versionCheck must have internal -// linkage (so it does NOT collide between versions); if we're trying to have -// multiple versions loaded at the same time, they must each run their copy -// of versionCheck, but share the PRODUCT##Version variable. -#define FOLLY_VERSION_CHECK(PRODUCT, VERSION) \ - const char* PRODUCT##Version = VERSION; \ - namespace { \ - FOLLY_VERSION_CHECK_PRIORITY(void, versionCheck) { \ - if (strcmp(PRODUCT##Version, VERSION)) { \ - fprintf(stderr, \ - "Invalid %s version: desired [%s], currently loaded [%s]\n", \ - FB_STRINGIZE(PRODUCT), PRODUCT##Version, VERSION); \ - abort(); \ - } \ - } \ - } diff --git a/folly/concurrency/CacheLocality.h b/folly/concurrency/CacheLocality.h index 38f5557c..457e107a 100644 --- a/folly/concurrency/CacheLocality.h +++ b/folly/concurrency/CacheLocality.h @@ -33,9 +33,9 @@ #include #include #include -#include #include #include +#include namespace folly { diff --git a/folly/detail/MemoryIdler.h b/folly/detail/MemoryIdler.h index fcbde002..13fe453e 100644 --- a/folly/detail/MemoryIdler.h +++ b/folly/detail/MemoryIdler.h @@ -21,9 +21,9 @@ #include #include -#include #include #include +#include namespace folly { diff --git a/folly/executors/ThreadedExecutor.cpp b/folly/executors/ThreadedExecutor.cpp index e19ecf4f..badfc549 100644 --- a/folly/executors/ThreadedExecutor.cpp +++ b/folly/executors/ThreadedExecutor.cpp @@ -20,8 +20,8 @@ #include -#include #include +#include namespace folly { diff --git a/folly/executors/thread_factory/NamedThreadFactory.h b/folly/executors/thread_factory/NamedThreadFactory.h index 2b34be9e..12cb7a7e 100644 --- a/folly/executors/thread_factory/NamedThreadFactory.h +++ b/folly/executors/thread_factory/NamedThreadFactory.h @@ -22,8 +22,8 @@ #include #include -#include #include +#include namespace folly { diff --git a/folly/experimental/FunctionScheduler.cpp b/folly/experimental/FunctionScheduler.cpp index cee5a4c4..61538106 100644 --- a/folly/experimental/FunctionScheduler.cpp +++ b/folly/experimental/FunctionScheduler.cpp @@ -21,7 +21,7 @@ #include #include #include -#include +#include using std::chrono::milliseconds; using std::chrono::steady_clock; diff --git a/folly/experimental/ThreadedRepeatingFunctionRunner.cpp b/folly/experimental/ThreadedRepeatingFunctionRunner.cpp index 82ab4081..7c478d1a 100644 --- a/folly/experimental/ThreadedRepeatingFunctionRunner.cpp +++ b/folly/experimental/ThreadedRepeatingFunctionRunner.cpp @@ -15,7 +15,7 @@ */ #include "folly/experimental/ThreadedRepeatingFunctionRunner.h" -#include +#include #include #include diff --git a/folly/experimental/io/HugePageUtil.cpp b/folly/experimental/io/HugePageUtil.cpp index 8701856b..25735bfe 100644 --- a/folly/experimental/io/HugePageUtil.cpp +++ b/folly/experimental/io/HugePageUtil.cpp @@ -18,10 +18,10 @@ #include #include -#include #include #include #include +#include DEFINE_bool(cp, false, "Copy file"); diff --git a/folly/experimental/logging/LogMessage.cpp b/folly/experimental/logging/LogMessage.cpp index 3ec39603..7b4bd253 100644 --- a/folly/experimental/logging/LogMessage.cpp +++ b/folly/experimental/logging/LogMessage.cpp @@ -15,7 +15,7 @@ */ #include -#include +#include using std::chrono::system_clock; diff --git a/folly/experimental/observer/detail/ObserverManager.cpp b/folly/experimental/observer/detail/ObserverManager.cpp index 88372ad6..e9312db5 100644 --- a/folly/experimental/observer/detail/ObserverManager.cpp +++ b/folly/experimental/observer/detail/ObserverManager.cpp @@ -20,8 +20,8 @@ #include #include #include -#include #include +#include namespace folly { namespace observer_detail { diff --git a/folly/io/RecordIO.h b/folly/io/RecordIO.h index 856e329f..45381080 100644 --- a/folly/io/RecordIO.h +++ b/folly/io/RecordIO.h @@ -31,9 +31,9 @@ #include #include -#include #include #include +#include namespace folly { diff --git a/folly/io/async/EventBase.cpp b/folly/io/async/EventBase.cpp index 56f92c8d..d138c336 100644 --- a/folly/io/async/EventBase.cpp +++ b/folly/io/async/EventBase.cpp @@ -27,10 +27,10 @@ #include #include -#include #include #include #include +#include namespace folly { diff --git a/folly/io/async/SSLContext.cpp b/folly/io/async/SSLContext.cpp index d0134178..498d8dfb 100644 --- a/folly/io/async/SSLContext.cpp +++ b/folly/io/async/SSLContext.cpp @@ -21,8 +21,8 @@ #include #include #include -#include #include +#include // --------------------------------------------------------------------- // SSLContext implementation diff --git a/folly/io/async/ScopedEventBaseThread.cpp b/folly/io/async/ScopedEventBaseThread.cpp index 1e21aad7..7ffaae62 100644 --- a/folly/io/async/ScopedEventBaseThread.cpp +++ b/folly/io/async/ScopedEventBaseThread.cpp @@ -20,8 +20,8 @@ #include #include -#include #include +#include using namespace std; diff --git a/folly/io/async/test/EventBaseThreadTest.cpp b/folly/io/async/test/EventBaseThreadTest.cpp index 96939e33..e804501e 100644 --- a/folly/io/async/test/EventBaseThreadTest.cpp +++ b/folly/io/async/test/EventBaseThreadTest.cpp @@ -19,9 +19,9 @@ #include #include -#include #include #include +#include using namespace std; using namespace std::chrono; diff --git a/folly/io/async/test/ScopedEventBaseThreadTest.cpp b/folly/io/async/test/ScopedEventBaseThreadTest.cpp index 79dbe6ff..6f1d2266 100644 --- a/folly/io/async/test/ScopedEventBaseThreadTest.cpp +++ b/folly/io/async/test/ScopedEventBaseThreadTest.cpp @@ -21,9 +21,9 @@ #include #include -#include #include #include +#include using namespace std; using namespace std::chrono; diff --git a/folly/io/async/test/TimeUtil.cpp b/folly/io/async/test/TimeUtil.cpp index 9340a9d6..94c70374 100644 --- a/folly/io/async/test/TimeUtil.cpp +++ b/folly/io/async/test/TimeUtil.cpp @@ -33,8 +33,8 @@ #include #include -#include #include +#include #include diff --git a/folly/system/MemoryMapping.cpp b/folly/system/MemoryMapping.cpp new file mode 100644 index 00000000..59d09f49 --- /dev/null +++ b/folly/system/MemoryMapping.cpp @@ -0,0 +1,407 @@ +/* + * Copyright 2017 Facebook, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include +#include +#include + +#include +#include +#include + +#ifdef __linux__ +#include +#endif + +#include +#include +#include + +static constexpr ssize_t kDefaultMlockChunkSize = +#ifndef _MSC_VER + // Linux implementations of unmap/mlock/munlock take a kernel + // semaphore and block other threads from doing other memory + // operations. Split the operations in chunks. + (1 << 20) // 1MB +#else // _MSC_VER + // MSVC doesn't have this problem, and calling munmap many times + // with the same address is a bad idea with the windows implementation. + (-1) +#endif // _MSC_VER + ; + +DEFINE_int64(mlock_chunk_size, kDefaultMlockChunkSize, + "Maximum bytes to mlock/munlock/munmap at once " + "(will be rounded up to PAGESIZE). Ignored if negative."); + +#ifndef MAP_POPULATE +#define MAP_POPULATE 0 +#endif + +namespace folly { + +MemoryMapping::MemoryMapping(MemoryMapping&& other) noexcept { + swap(other); +} + +MemoryMapping::MemoryMapping(File file, off_t offset, off_t length, + Options options) + : file_(std::move(file)), + options_(std::move(options)) { + CHECK(file_); + init(offset, length); +} + +MemoryMapping::MemoryMapping(const char* name, off_t offset, off_t length, + Options options) + : MemoryMapping(File(name, options.writable ? O_RDWR : O_RDONLY), + offset, + length, + options) { } + +MemoryMapping::MemoryMapping(int fd, off_t offset, off_t length, + Options options) + : MemoryMapping(File(fd), offset, length, options) { } + +MemoryMapping::MemoryMapping(AnonymousType, off_t length, Options options) + : options_(std::move(options)) { + init(0, length); +} + +namespace { + +#ifdef __linux__ +void getDeviceOptions(dev_t device, off_t& pageSize, bool& autoExtend) { + auto ps = getHugePageSizeForDevice(device); + if (ps) { + pageSize = ps->size; + autoExtend = true; + } +} +#else +inline void getDeviceOptions(dev_t, off_t&, bool&) {} +#endif + +} // namespace + +void MemoryMapping::init(off_t offset, off_t length) { + const bool grow = options_.grow; + const bool anon = !file_; + CHECK(!(grow && anon)); + + off_t& pageSize = options_.pageSize; + + struct stat st; + + // On Linux, hugetlbfs file systems don't require ftruncate() to grow the + // file, and (on kernels before 2.6.24) don't even allow it. Also, the file + // size is always a multiple of the page size. + bool autoExtend = false; + + if (!anon) { + // Stat the file + CHECK_ERR(fstat(file_.fd(), &st)); + + if (pageSize == 0) { + getDeviceOptions(st.st_dev, pageSize, autoExtend); + } + } else { + DCHECK(!file_); + DCHECK_EQ(offset, 0); + CHECK_EQ(pageSize, 0); + CHECK_GE(length, 0); + } + + if (pageSize == 0) { + pageSize = off_t(sysconf(_SC_PAGESIZE)); + } + + CHECK_GT(pageSize, 0); + CHECK_EQ(pageSize & (pageSize - 1), 0); // power of two + CHECK_GE(offset, 0); + + // Round down the start of the mapped region + off_t skipStart = offset % pageSize; + offset -= skipStart; + + mapLength_ = length; + if (mapLength_ != -1) { + mapLength_ += skipStart; + + // Round up the end of the mapped region + mapLength_ = (mapLength_ + pageSize - 1) / pageSize * pageSize; + } + + off_t remaining = anon ? length : st.st_size - offset; + + if (mapLength_ == -1) { + length = mapLength_ = remaining; + } else { + if (length > remaining) { + if (grow) { + if (!autoExtend) { + PCHECK(0 == ftruncate(file_.fd(), offset + length)) + << "ftruncate() failed, couldn't grow file to " + << offset + length; + remaining = length; + } else { + // Extend mapping to multiple of page size, don't use ftruncate + remaining = mapLength_; + } + } else { + length = remaining; + } + } + if (mapLength_ > remaining) { + mapLength_ = remaining; + } + } + + if (length == 0) { + mapLength_ = 0; + mapStart_ = nullptr; + } else { + int flags = options_.shared ? MAP_SHARED : MAP_PRIVATE; + if (anon) { + flags |= MAP_ANONYMOUS; + } + if (options_.prefault) { + flags |= MAP_POPULATE; + } + + // The standard doesn't actually require PROT_NONE to be zero... + int prot = PROT_NONE; + if (options_.readable || options_.writable) { + prot = ((options_.readable ? PROT_READ : 0) | + (options_.writable ? PROT_WRITE : 0)); + } + + unsigned char* start = static_cast(mmap( + options_.address, size_t(mapLength_), prot, flags, file_.fd(), offset)); + PCHECK(start != MAP_FAILED) + << " offset=" << offset + << " length=" << mapLength_; + mapStart_ = start; + data_.reset(start + skipStart, size_t(length)); + } +} + +namespace { + +off_t memOpChunkSize(off_t length, off_t pageSize) { + off_t chunkSize = length; + if (FLAGS_mlock_chunk_size <= 0) { + return chunkSize; + } + + chunkSize = off_t(FLAGS_mlock_chunk_size); + off_t r = chunkSize % pageSize; + if (r) { + chunkSize += (pageSize - r); + } + return chunkSize; +} + +/** + * Run @op in chunks over the buffer @mem of @bufSize length. + * + * Return: + * - success: true + amountSucceeded == bufSize (op success on whole buffer) + * - failure: false + amountSucceeded == nr bytes on which op succeeded. + */ +bool memOpInChunks(std::function op, + void* mem, size_t bufSize, off_t pageSize, + size_t& amountSucceeded) { + // Linux' unmap/mlock/munlock take a kernel semaphore and block other threads + // from doing other memory operations. If the size of the buffer is big the + // semaphore can be down for seconds (for benchmarks see + // http://kostja-osipov.livejournal.com/42963.html). Doing the operations in + // chunks breaks the locking into intervals and lets other threads do memory + // operations of their own. + + size_t chunkSize = size_t(memOpChunkSize(off_t(bufSize), pageSize)); + + char* addr = static_cast(mem); + amountSucceeded = 0; + + while (amountSucceeded < bufSize) { + size_t size = std::min(chunkSize, bufSize - amountSucceeded); + if (op(addr + amountSucceeded, size) != 0) { + return false; + } + amountSucceeded += size; + } + + return true; +} + +} // namespace + +bool MemoryMapping::mlock(LockMode lock) { + size_t amountSucceeded = 0; + locked_ = memOpInChunks( + ::mlock, + mapStart_, + size_t(mapLength_), + options_.pageSize, + amountSucceeded); + if (locked_) { + return true; + } + + auto msg = + folly::format("mlock({}) failed at {}", mapLength_, amountSucceeded); + if (lock == LockMode::TRY_LOCK && errno == EPERM) { + PLOG(WARNING) << msg; + } else if (lock == LockMode::TRY_LOCK && errno == ENOMEM) { + VLOG(1) << msg; + } else { + PLOG(FATAL) << msg; + } + + // only part of the buffer was mlocked, unlock it back + if (!memOpInChunks(::munlock, mapStart_, amountSucceeded, options_.pageSize, + amountSucceeded)) { + PLOG(WARNING) << "munlock()"; + } + + return false; +} + +void MemoryMapping::munlock(bool dontneed) { + if (!locked_) { + return; + } + + size_t amountSucceeded = 0; + if (!memOpInChunks( + ::munlock, + mapStart_, + size_t(mapLength_), + options_.pageSize, + amountSucceeded)) { + PLOG(WARNING) << "munlock()"; + } + if (mapLength_ && dontneed && + ::madvise(mapStart_, size_t(mapLength_), MADV_DONTNEED)) { + PLOG(WARNING) << "madvise()"; + } + locked_ = false; +} + +void MemoryMapping::hintLinearScan() { + advise(MADV_SEQUENTIAL); +} + +MemoryMapping::~MemoryMapping() { + if (mapLength_) { + size_t amountSucceeded = 0; + if (!memOpInChunks( + ::munmap, + mapStart_, + size_t(mapLength_), + options_.pageSize, + amountSucceeded)) { + PLOG(FATAL) << folly::format("munmap({}) failed at {}", + mapLength_, amountSucceeded); + } + } +} + +void MemoryMapping::advise(int advice) const { + advise(advice, 0, size_t(mapLength_)); +} + +void MemoryMapping::advise(int advice, size_t offset, size_t length) const { + CHECK_LE(offset + length, size_t(mapLength_)) + << " offset: " << offset + << " length: " << length + << " mapLength_: " << mapLength_; + + // Include the entire start page: round down to page boundary. + const auto offMisalign = offset % options_.pageSize; + offset -= offMisalign; + length += offMisalign; + + // Round the last page down to page boundary. + if (offset + length != size_t(mapLength_)) { + length -= length % options_.pageSize; + } + + if (length == 0) { + return; + } + + char* mapStart = static_cast(mapStart_) + offset; + PLOG_IF(WARNING, ::madvise(mapStart, length, advice)) << "madvise"; +} + +MemoryMapping& MemoryMapping::operator=(MemoryMapping other) { + swap(other); + return *this; +} + +void MemoryMapping::swap(MemoryMapping& other) noexcept { + using std::swap; + swap(this->file_, other.file_); + swap(this->mapStart_, other.mapStart_); + swap(this->mapLength_, other.mapLength_); + swap(this->options_, other.options_); + swap(this->locked_, other.locked_); + swap(this->data_, other.data_); +} + +void swap(MemoryMapping& a, MemoryMapping& b) noexcept { a.swap(b); } + +void alignedForwardMemcpy(void* dst, const void* src, size_t size) { + assert(reinterpret_cast(src) % alignof(unsigned long) == 0); + assert(reinterpret_cast(dst) % alignof(unsigned long) == 0); + + auto srcl = static_cast(src); + auto dstl = static_cast(dst); + + while (size >= sizeof(unsigned long)) { + *dstl++ = *srcl++; + size -= sizeof(unsigned long); + } + + auto srcc = reinterpret_cast(srcl); + auto dstc = reinterpret_cast(dstl); + + while (size != 0) { + *dstc++ = *srcc++; + --size; + } +} + +void mmapFileCopy(const char* src, const char* dest, mode_t mode) { + MemoryMapping srcMap(src); + srcMap.hintLinearScan(); + + MemoryMapping destMap( + File(dest, O_RDWR | O_CREAT | O_TRUNC, mode), + 0, + off_t(srcMap.range().size()), + MemoryMapping::writable()); + + alignedForwardMemcpy(destMap.writableRange().data(), + srcMap.range().data(), + srcMap.range().size()); +} + +} // namespace folly diff --git a/folly/system/MemoryMapping.h b/folly/system/MemoryMapping.h new file mode 100644 index 00000000..38220632 --- /dev/null +++ b/folly/system/MemoryMapping.h @@ -0,0 +1,251 @@ +/* + * Copyright 2017 Facebook, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include + +#include +#include + +namespace folly { + +/** + * Maps files in memory (read-only). + * + * @author Tudor Bosman (tudorb@fb.com) + */ +class MemoryMapping : boost::noncopyable { + public: + /** + * Lock the pages in memory? + * TRY_LOCK = try to lock, log warning if permission denied + * MUST_LOCK = lock, fail assertion if permission denied. + */ + enum class LockMode { + TRY_LOCK, + MUST_LOCK + }; + /** + * Map a portion of the file indicated by filename in memory, causing a CHECK + * failure on error. + * + * By default, map the whole file. length=-1: map from offset to EOF. + * Unlike the mmap() system call, offset and length don't need to be + * page-aligned. length is clipped to the end of the file if it's too large. + * + * The mapping will be destroyed (and the memory pointed-to by data() will + * likely become inaccessible) when the MemoryMapping object is destroyed. + */ + struct Options { + Options() {} + + // Convenience methods; return *this for chaining. + Options& setPageSize(off_t v) { pageSize = v; return *this; } + Options& setShared(bool v) { shared = v; return *this; } + Options& setPrefault(bool v) { prefault = v; return *this; } + Options& setReadable(bool v) { readable = v; return *this; } + Options& setWritable(bool v) { writable = v; return *this; } + Options& setGrow(bool v) { grow = v; return *this; } + + // Page size. 0 = use appropriate page size. + // (On Linux, we use a huge page size if the file is on a hugetlbfs + // file system, and the default page size otherwise) + off_t pageSize = 0; + + // If shared (default), the memory mapping is shared with other processes + // mapping the same file (or children); if not shared (private), each + // process has its own mapping. Changes in writable, private mappings are + // not reflected to the underlying file. See the discussion of + // MAP_PRIVATE vs MAP_SHARED in the mmap(2) manual page. + bool shared = true; + + // Populate page tables; subsequent accesses should not be blocked + // by page faults. This is a hint, as it may not be supported. + bool prefault = false; + + // Map the pages readable. Note that mapping pages without read permissions + // is not universally supported (not supported on hugetlbfs on Linux, for + // example) + bool readable = true; + + // Map the pages writable. + bool writable = false; + + // When mapping a file in writable mode, grow the file to the requested + // length (using ftruncate()) before mapping; if false, truncate the + // mapping to the actual file size instead. + bool grow = false; + + // Fix map at this address, if not nullptr. Must be aligned to a multiple + // of the appropriate page size. + void* address = nullptr; + }; + + // Options to emulate the old WritableMemoryMapping: readable and writable, + // allow growing the file if mapping past EOF. + static Options writable() { + return Options().setWritable(true).setGrow(true); + } + + enum AnonymousType { + kAnonymous + }; + + /** + * Create an anonymous mapping. + */ + MemoryMapping(AnonymousType, off_t length, Options options=Options()); + + explicit MemoryMapping(File file, + off_t offset=0, + off_t length=-1, + Options options=Options()); + + explicit MemoryMapping(const char* name, + off_t offset=0, + off_t length=-1, + Options options=Options()); + + explicit MemoryMapping(int fd, + off_t offset=0, + off_t length=-1, + Options options=Options()); + + MemoryMapping(MemoryMapping&&) noexcept; + + ~MemoryMapping(); + + MemoryMapping& operator=(MemoryMapping); + + void swap(MemoryMapping& other) noexcept; + + /** + * Lock the pages in memory + */ + bool mlock(LockMode lock); + + /** + * Unlock the pages. + * If dontneed is true, the kernel is instructed to release these pages + * (per madvise(MADV_DONTNEED)). + */ + void munlock(bool dontneed = false); + + /** + * Hint that these pages will be scanned linearly. + * madvise(MADV_SEQUENTIAL) + */ + void hintLinearScan(); + + /** + * Advise the kernel about memory access. + */ + void advise(int advice) const; + void advise(int advice, size_t offset, size_t length) const; + + /** + * A bitwise cast of the mapped bytes as range of values. Only intended for + * use with POD or in-place usable types. + */ + template + Range asRange() const { + size_t count = data_.size() / sizeof(T); + return Range(static_cast( + static_cast(data_.data())), + count); + } + + /** + * A range of bytes mapped by this mapping. + */ + ByteRange range() const { + return data_; + } + + /** + * A bitwise cast of the mapped bytes as range of mutable values. Only + * intended for use with POD or in-place usable types. + */ + template + Range asWritableRange() const { + DCHECK(options_.writable); // you'll segfault anyway... + size_t count = data_.size() / sizeof(T); + return Range(static_cast( + static_cast(data_.data())), + count); + } + + /** + * A range of mutable bytes mapped by this mapping. + */ + MutableByteRange writableRange() const { + DCHECK(options_.writable); // you'll segfault anyway... + return data_; + } + + /** + * Return the memory area where the file was mapped. + * Deprecated; use range() instead. + */ + StringPiece data() const { + return asRange(); + } + + bool mlocked() const { + return locked_; + } + + int fd() const { return file_.fd(); } + + private: + MemoryMapping(); + + enum InitFlags { + kGrow = 1 << 0, + kAnon = 1 << 1, + }; + void init(off_t offset, off_t length); + + File file_; + void* mapStart_ = nullptr; + off_t mapLength_ = 0; + Options options_; + bool locked_ = false; + MutableByteRange data_; +}; + +void swap(MemoryMapping&, MemoryMapping&) noexcept; + +/** + * A special case of memcpy() that always copies memory forwards. + * (libc's memcpy() is allowed to copy memory backwards, and will do so + * when using SSSE3 instructions). + * + * Assumes src and dest are aligned to alignof(unsigned long). + * + * Useful when copying from/to memory mappings after hintLinearScan(); + * copying backwards renders any prefetching useless (even harmful). + */ +void alignedForwardMemcpy(void* dest, const void* src, size_t size); + +/** + * Copy a file using mmap(). Overwrites dest. + */ +void mmapFileCopy(const char* src, const char* dest, mode_t mode = 0666); + +} // namespace folly diff --git a/folly/system/Shell.cpp b/folly/system/Shell.cpp new file mode 100644 index 00000000..fbc075b0 --- /dev/null +++ b/folly/system/Shell.cpp @@ -0,0 +1,33 @@ +/* + * Copyright 2017 Facebook, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +namespace folly { + +std::string shellQuote(StringPiece argument) { + std::string quoted = "'"; + for (auto c : argument) { + if (c == '\'') { + quoted += "'\\''"; + } else { + quoted += c; + } + } + return quoted + "'"; +} + +} // namespace folly diff --git a/folly/system/Shell.h b/folly/system/Shell.h new file mode 100644 index 00000000..cefe2ddb --- /dev/null +++ b/folly/system/Shell.h @@ -0,0 +1,93 @@ +/* + * Copyright 2017 Facebook, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * `Shell` provides a collection of functions to use with `Subprocess` that make + * it easier to safely run processes in a unix shell. + * + * Note: use this rarely and carefully. By default you should use `Subprocess` + * with a vector of arguments. + */ + +#pragma once + +#include +#include + +#include +#include +#include + +namespace folly { + +/** + * Quotes an argument to make it suitable for use as shell command arguments. + */ +std::string shellQuote(StringPiece argument); + +namespace detail { +template +std::vector shellify( + StringPiece format, + Arguments&&... arguments) { + auto command = sformat( + format, + shellQuote(to(std::forward(arguments)))...); + return {"/bin/sh", "-c", command}; +} + +struct ShellCmdFormat { + StringPiece format; + template + std::vector operator()(Arguments&&... arguments) const { + return ::folly::detail::shellify( + format, std::forward(arguments)...); + } +}; + +} // namespace detail + +inline namespace literals { +inline namespace shell_literals { +constexpr detail::ShellCmdFormat operator"" _shellify( + char const* name, + std::size_t length) { + return {folly::StringPiece(name, length)}; +} +} // inline namespace shell_literals +} // inline namespace literals + +/** + * Create argument array for `Subprocess()` for a process running in a + * shell. + * + * The shell to use is always going to be `/bin/sh`. + * + * This is deprecated in favour of the user-defined-literal `_shellify` + * from namespace `folly::shell_literals` because that requires that the format + * string is a compile-time constant which can be inspected during code reviews + */ +template +FOLLY_DEPRECATED( + "Use `\"command {} {} ...\"_shellify(argument1, argument2 ...)` from " + "namespace `folly::literals::shell_literals`") +std::vector shellify( + StringPiece format, + Arguments&&... arguments) { + return detail::shellify(format, std::forward(arguments)...); +} + +} // namespace folly diff --git a/folly/system/ThreadId.h b/folly/system/ThreadId.h new file mode 100644 index 00000000..18b82cd7 --- /dev/null +++ b/folly/system/ThreadId.h @@ -0,0 +1,91 @@ +/* + * Copyright 2017 Facebook, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include + +#include +#include +#include +#include + +namespace folly { + +/** + * Get a process-specific identifier for the current thread. + * + * The return value will uniquely identify the thread within the current + * process. + * + * Note that the return value does not necessarily correspond to an operating + * system thread ID. The return value is also only unique within the current + * process: getCurrentThreadID() may return the same value for two concurrently + * running threads in separate processes. + * + * The thread ID may be reused once the thread it corresponds to has been + * joined. + */ +inline uint64_t getCurrentThreadID() { +#if __APPLE__ + return uint64_t(pthread_mach_thread_np(pthread_self())); +#elif _WIN32 + return uint64_t(GetCurrentThreadId()); +#else + return uint64_t(pthread_self()); +#endif +} + +/** + * Get the operating-system level thread ID for the current thread. + * + * The returned value will uniquely identify this thread on the system. + * + * This makes it more suitable for logging or displaying in user interfaces + * than the result of getCurrentThreadID(). + * + * There are some potential caveats about this API, however: + * + * - In theory there is no guarantee that application threads map one-to-one to + * kernel threads. An application threading implementation could potentially + * share one OS thread across multiple application threads, and/or it could + * potentially move application threads between different OS threads over + * time. However, in practice all of the platforms we currently support have + * a one-to-one mapping between userspace threads and operating system + * threads. + * + * - This API may also be slightly slower than getCurrentThreadID() on some + * platforms. This API may require a system call, where getCurrentThreadID() + * may only need to read thread-local memory. + * + * On Linux the returned value is a pid_t, and can be used in contexts + * requiring a thread pid_t. + * + * The thread ID may be reused once the thread it corresponds to has been + * joined. + */ +inline uint64_t getOSThreadID() { +#if __APPLE__ + uint64_t tid; + pthread_threadid_np(nullptr, &tid); + return tid; +#elif _WIN32 + return uint64_t(GetCurrentThreadId()); +#else + return uint64_t(syscall(FOLLY_SYS_gettid)); +#endif +} +} diff --git a/folly/system/ThreadName.cpp b/folly/system/ThreadName.cpp new file mode 100644 index 00000000..49793a91 --- /dev/null +++ b/folly/system/ThreadName.cpp @@ -0,0 +1,151 @@ +/* + * Copyright 2017 Facebook, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include + +#include +#include +#include + +namespace folly { + +// This looks a bit weird, but it's necessary to avoid +// having an undefined compiler function called. +#if defined(__GLIBC__) && !defined(__APPLE__) && !defined(__ANDROID__) +#if __GLIBC_PREREQ(2, 12) +// has pthread_setname_np(pthread_t, const char*) (2 params) +#define FOLLY_HAS_PTHREAD_SETNAME_NP_THREAD_NAME 1 +#endif +#endif + +#if defined(__APPLE__) && defined(__MAC_OS_X_VERSION_MIN_REQUIRED) +#if __MAC_OS_X_VERSION_MIN_REQUIRED >= 1060 +// has pthread_setname_np(const char*) (1 param) +#define FOLLY_HAS_PTHREAD_SETNAME_NP_NAME 1 +#endif +#endif + +namespace { + +#if FOLLY_HAVE_PTHREAD && !_WIN32 +pthread_t stdTidToPthreadId(std::thread::id tid) { + static_assert( + std::is_same::value, + "This assumes that the native handle type is pthread_t"); + static_assert( + sizeof(std::thread::native_handle_type) == sizeof(std::thread::id), + "This assumes std::thread::id is a thin wrapper around " + "std::thread::native_handle_type, but that doesn't appear to be true."); + // In most implementations, std::thread::id is a thin wrapper around + // std::thread::native_handle_type, which means we can do unsafe things to + // extract it. + pthread_t id; + std::memcpy(&id, &tid, sizeof(id)); + return id; +} +#endif + +} // namespace + +bool canSetCurrentThreadName() { +#if FOLLY_HAS_PTHREAD_SETNAME_NP_THREAD_NAME || \ + FOLLY_HAS_PTHREAD_SETNAME_NP_NAME + return true; +#else + return false; +#endif +} + +bool canSetOtherThreadName() { +#if FOLLY_HAS_PTHREAD_SETNAME_NP_THREAD_NAME + return true; +#else + return false; +#endif +} + +static constexpr size_t kMaxThreadNameLength = 16; + +Optional getThreadName(std::thread::id id) { +#if FOLLY_HAS_PTHREAD_SETNAME_NP_THREAD_NAME || \ + FOLLY_HAS_PTHREAD_SETNAME_NP_NAME + std::array buf; + if (pthread_getname_np(stdTidToPthreadId(id), buf.data(), buf.size()) != 0) { + return Optional(); + } + return make_optional(std::string(buf.data())); +#else + return Optional(); +#endif +} + +Optional getCurrentThreadName() { + return getThreadName(std::this_thread::get_id()); +} + +bool setThreadName(std::thread::id tid, StringPiece name) { +#if !FOLLY_HAVE_PTHREAD || _WIN32 + return false; +#else + name = name.subpiece(0, kMaxThreadNameLength - 1); + char buf[kMaxThreadNameLength] = {}; + std::memcpy(buf, name.data(), name.size()); + auto id = stdTidToPthreadId(tid); +#if FOLLY_HAS_PTHREAD_SETNAME_NP_THREAD_NAME + return 0 == pthread_setname_np(id, buf); +#elif FOLLY_HAS_PTHREAD_SETNAME_NP_NAME + // Since OS X 10.6 it is possible for a thread to set its own name, + // but not that of some other thread. + if (pthread_equal(pthread_self(), id)) { + return 0 == pthread_setname_np(buf); + } + return false; +#else + (void)id; + return false; +#endif +#endif +} + +#if FOLLY_HAVE_PTHREAD +bool setThreadName(pthread_t pid, StringPiece name) { +#if _WIN32 + // Not currently supported on Windows. + return false; +#else + static_assert( + std::is_same::value, + "This assumes that the native handle type is pthread_t"); + static_assert( + sizeof(std::thread::native_handle_type) == sizeof(std::thread::id), + "This assumes std::thread::id is a thin wrapper around " + "std::thread::native_handle_type, but that doesn't appear to be true."); + // In most implementations, std::thread::id is a thin wrapper around + // std::thread::native_handle_type, which means we can do unsafe things to + // extract it. + std::thread::id id; + std::memcpy(&id, &pid, sizeof(id)); + return setThreadName(id, name); +#endif +} +#endif + +bool setThreadName(StringPiece name) { + return setThreadName(std::this_thread::get_id(), name); +} +} diff --git a/folly/system/ThreadName.h b/folly/system/ThreadName.h new file mode 100644 index 00000000..ba7bde25 --- /dev/null +++ b/folly/system/ThreadName.h @@ -0,0 +1,66 @@ +/* + * Copyright 2017 Facebook, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include + +#include +#include +#include +#include + +namespace folly { + +/** + * This returns true if the current platform supports setting the name of the + * current thread. + */ +bool canSetCurrentThreadName(); + +/** + * This returns true if the current platform supports setting the name of + * threads other than the one currently executing. + */ +bool canSetOtherThreadName(); + +/** + * Get the name of the given thread, or nothing if an error occurs + * or the functionality is not available. + */ +Optional getThreadName(std::thread::id tid); + +/** + * Equivalent to getThreadName(std::this_thread::get_id()); + */ +Optional getCurrentThreadName(); + +/** + * Set the name of the given thread. + * Returns false on failure, if an error occurs or the functionality + * is not available. + */ +bool setThreadName(std::thread::id tid, StringPiece name); +#if FOLLY_HAVE_PTHREAD +bool setThreadName(pthread_t pid, StringPiece name); +#endif + +/** + * Equivalent to setThreadName(std::this_thread::get_id(), name); + */ +bool setThreadName(StringPiece name); +} diff --git a/folly/system/VersionCheck.cpp b/folly/system/VersionCheck.cpp new file mode 100644 index 00000000..be333d68 --- /dev/null +++ b/folly/system/VersionCheck.cpp @@ -0,0 +1,23 @@ +/* + * Copyright 2017 Facebook, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +namespace folly { namespace detail { + +FOLLY_VERSION_CHECK(folly, FOLLY_VERSION) + +}} // namespaces diff --git a/folly/system/VersionCheck.h b/folly/system/VersionCheck.h new file mode 100644 index 00000000..049556ef --- /dev/null +++ b/folly/system/VersionCheck.h @@ -0,0 +1,108 @@ +/* + * Copyright 2017 Facebook, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include + +#include +#include + +/** + * Check if the currently loaded version of a library is what you expect. + * + * It is possible for multiple versions of the same shared library to end up + * being loaded simultaneously in the same address space, usually with + * disastrous results. + * + * For example, let's say you have a shared library (foo) that doesn't keep + * binary compatbility between releases, and so each version is distributed as + * a SO with different SONAME. Let's say you build another shared library, bar + * that depends on version 1 of foo: libbar.so depends on libfoo1.so. + * Your main executable now (baz) depends on version 2 of foo, and also + * depends on bar: baz depends on libfoo2.so and libbar.so. + * + * At load time, baz loads libfoo2.so first, then libbar.so; libbar.so will + * load libfoo1.so, but, as this is normal dynamic loading (and not explicit + * dlopen calls with RTLD_DEEPBIND), any symbols from libfoo1.so that are + * also present in libfoo2.so will be satisfied from the (already loaded) + * libfoo2.so. + * + * But foo does not preserve binary compatibility between versions, so all + * hell breaks loose (the symbols from libfoo2.so are not necessarily direct + * replacements of the identically-named symbols in libfoo1.so). + * + * It is better to crash with a helpful error message instead, which is what + * this macro provides. FOLLY_VERSION_CHECK verifies at load time that + * the compiled-in version is the same as the currently loaded version. + * + * Usage: use this macro at namespace scope in a .cpp file (IMPORTANT: NOT + * in the unnamed namespace): + * + * FOLLY_VERSION_CHECK(mylib, "1") + * + * The first argument identifies your library; the second argument is a + * string literal containing the desired version string. + * + * In order to avoid changing the file for each version, the version string + * could be provided on the compiler command line with -D: + * + * FOLLY_VERSION_CHECK(mylib, MYLIB_VERSION) + * + * ... and then commpile your file with -DMYLIB_VERSION=\"1\" + */ + +#if defined(_MSC_VER) +// MSVC doesn't support constructor priorities. Just pray it works, I guess. +// We could implement a link-time mechanism for MSVC, +// via #pragma detect_mismatch but that would only handle +// static library linking. +# define FOLLY_VERSION_CHECK_PRIORITY(Ret, name) \ + __pragma(section(".CRT$XCU",read)) \ + static Ret __cdecl name(void); \ + __declspec(allocate(".CRT$XCU")) \ + Ret (__cdecl*name##_)(void) = name; \ + Ret __cdecl name() + +#elif defined(__APPLE__) +// OS X doesn't support constructor priorities. Just pray it works, I guess. +# define FOLLY_VERSION_CHECK_PRIORITY(Ret, name) \ + __attribute__((__constructor__)) Ret name() + +#else +# define FOLLY_VERSION_CHECK_PRIORITY(Ret, name) \ + __attribute__((__constructor__(101))) Ret name() +#endif + +// Note that this is carefully crafted: PRODUCT##Version must have external +// linkage (so it collides among versions), versionCheck must have internal +// linkage (so it does NOT collide between versions); if we're trying to have +// multiple versions loaded at the same time, they must each run their copy +// of versionCheck, but share the PRODUCT##Version variable. +#define FOLLY_VERSION_CHECK(PRODUCT, VERSION) \ + const char* PRODUCT##Version = VERSION; \ + namespace { \ + FOLLY_VERSION_CHECK_PRIORITY(void, versionCheck) { \ + if (strcmp(PRODUCT##Version, VERSION)) { \ + fprintf(stderr, \ + "Invalid %s version: desired [%s], currently loaded [%s]\n", \ + FB_STRINGIZE(PRODUCT), PRODUCT##Version, VERSION); \ + abort(); \ + } \ + } \ + } diff --git a/folly/system/test/MemoryMappingTest.cpp b/folly/system/test/MemoryMappingTest.cpp new file mode 100644 index 00000000..eb2d5d53 --- /dev/null +++ b/folly/system/test/MemoryMappingTest.cpp @@ -0,0 +1,179 @@ +/* + * Copyright 2017 Facebook, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include +#include +#include +#include +#include + +static constexpr double kSomeDouble = 3.14; + +namespace folly { + +TEST(MemoryMapping, Basic) { + File f = File::temporary(); + { + MemoryMapping m(File(f.fd()), 0, sizeof(double), MemoryMapping::writable()); + double* d = m.asWritableRange().data(); + *d = 37 * kSomeDouble; + } + { + MemoryMapping m(File(f.fd()), 0, 3); + EXPECT_EQ(0, m.asRange().size()); // not big enough + } + { + MemoryMapping m(File(f.fd()), 0, sizeof(double)); + const double* d = m.asRange().data(); + EXPECT_EQ(*d, 37 * kSomeDouble); + } +} + +TEST(MemoryMapping, Move) { + File f = File::temporary(); + { + MemoryMapping m( + File(f.fd()), 0, sizeof(double) * 2, MemoryMapping::writable()); + double* d = m.asWritableRange().data(); + d[0] = 37 * kSomeDouble; + MemoryMapping m2(std::move(m)); + double* d2 = m2.asWritableRange().data(); + d2[1] = 39 * kSomeDouble; + } + { + MemoryMapping m(File(f.fd()), 0, sizeof(double)); + const double* d = m.asRange().data(); + EXPECT_EQ(d[0], 37 * kSomeDouble); + MemoryMapping m2(std::move(m)); + const double* d2 = m2.asRange().data(); + EXPECT_EQ(d2[1], 39 * kSomeDouble); + } +} + +TEST(MemoryMapping, DoublyMapped) { + File f = File::temporary(); + // two mappings of the same memory, different addresses. + MemoryMapping mw(File(f.fd()), 0, sizeof(double), MemoryMapping::writable()); + MemoryMapping mr(File(f.fd()), 0, sizeof(double)); + + double* dw = mw.asWritableRange().data(); + const double* dr = mr.asRange().data(); + + // Show that it's truly the same value, even though the pointers differ + EXPECT_NE(dw, dr); + *dw = 42 * kSomeDouble; + EXPECT_EQ(*dr, 42 * kSomeDouble); + *dw = 43 * kSomeDouble; + EXPECT_EQ(*dr, 43 * kSomeDouble); +} + +namespace { + +void writeStringToFileOrDie(const std::string& str, int fd) { + const char* b = str.c_str(); + size_t count = str.size(); + ssize_t total_bytes = 0; + ssize_t r; + do { + r = write(fd, b, count); + if (r == -1) { + if (errno == EINTR) { + continue; + } + PCHECK(r) << "write"; + } + + total_bytes += r; + b += r; + count -= r; + } while (r != 0 && count); +} + +} // namespace + +TEST(MemoryMapping, Simple) { + File f = File::temporary(); + writeStringToFileOrDie("hello", f.fd()); + + { + MemoryMapping m(File(f.fd())); + EXPECT_EQ("hello", m.data()); + } + { + MemoryMapping m(File(f.fd()), 1, 2); + EXPECT_EQ("el", m.data()); + } +} + +TEST(MemoryMapping, LargeFile) { + std::string fileData; + size_t fileSize = sysconf(_SC_PAGESIZE) * 3 + 10; + fileData.reserve(fileSize); + for (size_t i = 0; i < fileSize; i++) { + fileData.push_back(0xff & Random::rand32()); + } + + File f = File::temporary(); + writeStringToFileOrDie(fileData, f.fd()); + + { + MemoryMapping m(File(f.fd())); + EXPECT_EQ(fileData, m.data()); + } + { + size_t size = sysconf(_SC_PAGESIZE) * 2; + StringPiece s(fileData.data() + 9, size - 9); + MemoryMapping m(File(f.fd()), 9, size - 9); + EXPECT_EQ(s.toString(), m.data()); + } +} + +TEST(MemoryMapping, ZeroLength) { + File f = File::temporary(); + MemoryMapping m(File(f.fd())); + EXPECT_TRUE(m.mlock(MemoryMapping::LockMode::MUST_LOCK)); + EXPECT_TRUE(m.mlocked()); + EXPECT_EQ(0, m.data().size()); +} + +TEST(MemoryMapping, Advise) { + File f = File::temporary(); + size_t kPageSize = 4096; + size_t size = kPageSize + 10; // unaligned file size + PCHECK(ftruncateNoInt(f.fd(), size) == 0) << size; + + MemoryMapping m(File(f.fd())); + + // NOTE: advise crashes on bad input. + + m.advise(MADV_NORMAL, 0, kPageSize); + m.advise(MADV_NORMAL, 1, kPageSize); + m.advise(MADV_NORMAL, 0, 2); + m.advise(MADV_NORMAL, 1, 2); + + m.advise(MADV_NORMAL, kPageSize, 0); + m.advise(MADV_NORMAL, kPageSize, 1); + m.advise(MADV_NORMAL, kPageSize, size - kPageSize); + + auto off = kPageSize + 1; + m.advise(MADV_NORMAL, off, size - off); + + EXPECT_DEATH(m.advise(MADV_NORMAL, off, size - off + 1), ""); +} + +} // namespace folly diff --git a/folly/system/test/ShellTest.cpp b/folly/system/test/ShellTest.cpp new file mode 100644 index 00000000..42978693 --- /dev/null +++ b/folly/system/test/ShellTest.cpp @@ -0,0 +1,60 @@ +/* + * Copyright 2017 Facebook, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include +#include + +using namespace folly; + +TEST(Shell, ShellQuote) { + EXPECT_EQ(shellQuote("a"), "'a'"); + EXPECT_EQ(shellQuote("a'b"), "'a'\\''b'"); + EXPECT_EQ(shellQuote("a\"b"), "'a\"b'"); +} + +TEST(Shell, Shellify) { + auto command = "rm -rf /"_shellify(); + EXPECT_EQ(command[0], "/bin/sh"); + EXPECT_EQ(command[1], "-c"); + EXPECT_EQ(command[2], "rm -rf /"); + + command = "rm -rf {}"_shellify("someFile.txt"); + EXPECT_EQ(command[2], "rm -rf 'someFile.txt'"); + + command = "rm -rf {}"_shellify(5); + EXPECT_EQ(command[2], "rm -rf '5'"); + + command = "ls {}"_shellify("blah'; rm -rf /"); + EXPECT_EQ(command[2], "ls 'blah'\\''; rm -rf /'"); +} + +TEST(Shell, Shellify_deprecated) { + auto command = shellify("rm -rf /"); + EXPECT_EQ(command[0], "/bin/sh"); + EXPECT_EQ(command[1], "-c"); + EXPECT_EQ(command[2], "rm -rf /"); + + command = shellify("rm -rf {}", "someFile.txt"); + EXPECT_EQ(command[2], "rm -rf 'someFile.txt'"); + + command = shellify("rm -rf {}", 5); + EXPECT_EQ(command[2], "rm -rf '5'"); + + command = shellify("ls {}", "blah'; rm -rf /"); + EXPECT_EQ(command[2], "ls 'blah'\\''; rm -rf /'"); +} diff --git a/folly/system/test/ThreadIdTest.cpp b/folly/system/test/ThreadIdTest.cpp new file mode 100644 index 00000000..fcece811 --- /dev/null +++ b/folly/system/test/ThreadIdTest.cpp @@ -0,0 +1,40 @@ +/* + * Copyright 2017 Facebook, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Make sure we include ThreadId.h before anything else. +// There is no ThreadId.cpp file, so this test is the only thing that verifies +// that ThreadId.h compiles by itself when included first. +#include + +#include + +#include + +TEST(ThreadId, getCurrentID) { + auto thisThreadID = folly::getCurrentThreadID(); + uint64_t otherThreadID; + std::thread otherThread{[&] { otherThreadID = folly::getCurrentThreadID(); }}; + otherThread.join(); + EXPECT_NE(thisThreadID, otherThreadID); +} + +TEST(ThreadId, getOSThreadID) { + auto thisThreadID = folly::getOSThreadID(); + uint64_t otherThreadID; + std::thread otherThread{[&] { otherThreadID = folly::getOSThreadID(); }}; + otherThread.join(); + EXPECT_NE(thisThreadID, otherThreadID); +} diff --git a/folly/system/test/ThreadNameTest.cpp b/folly/system/test/ThreadNameTest.cpp new file mode 100644 index 00000000..9dce6bd9 --- /dev/null +++ b/folly/system/test/ThreadNameTest.cpp @@ -0,0 +1,74 @@ +/* + * Copyright 2017 Facebook, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include +#include +#include +#include + +using namespace std; +using namespace folly; + +namespace { + +const bool expectedSetOtherThreadNameResult = folly::canSetOtherThreadName(); +const bool expectedSetSelfThreadNameResult = folly::canSetCurrentThreadName(); +constexpr StringPiece kThreadName{"rockin-thread"}; + +} // namespace + +TEST(ThreadName, getCurrentThreadName) { + thread th([] { + EXPECT_EQ(expectedSetSelfThreadNameResult, setThreadName(kThreadName)); + if (expectedSetSelfThreadNameResult) { + EXPECT_EQ(kThreadName.toString(), *getCurrentThreadName()); + } + }); + SCOPE_EXIT { th.join(); }; +} + +TEST(ThreadName, setThreadName_other_pthread) { + Baton<> handle_set; + Baton<> let_thread_end; + pthread_t handle; + thread th([&] { + handle = pthread_self(); + handle_set.post(); + let_thread_end.wait(); + }); + SCOPE_EXIT { th.join(); }; + handle_set.wait(); + SCOPE_EXIT { let_thread_end.post(); }; + EXPECT_EQ( + expectedSetOtherThreadNameResult, setThreadName(handle, kThreadName)); +} + +TEST(ThreadName, setThreadName_other_id) { + Baton<> let_thread_end; + thread th([&] { + let_thread_end.wait(); + }); + SCOPE_EXIT { th.join(); }; + SCOPE_EXIT { let_thread_end.post(); }; + EXPECT_EQ( + expectedSetOtherThreadNameResult, + setThreadName(th.get_id(), kThreadName)); + if (expectedSetOtherThreadNameResult) { + EXPECT_EQ(*getThreadName(th.get_id()), kThreadName); + } +} diff --git a/folly/test/Makefile.am b/folly/test/Makefile.am index a2f58830..31e8b74b 100644 --- a/folly/test/Makefile.am +++ b/folly/test/Makefile.am @@ -102,7 +102,7 @@ TESTS += fbstring_test_using_jemalloc thread_cached_int_test_SOURCES = ThreadCachedIntTest.cpp thread_cached_int_test_LDADD = libfollytestmain.la $(top_builddir)/libfollybenchmark.la -thread_id_test_SOURCES = ThreadIdTest.cpp +thread_id_test_SOURCES = system/test/ThreadIdTest.cpp thread_id_test_LDADD = libfollytestmain.la TESTS += thread_id_test @@ -239,7 +239,7 @@ token_bucket_test_SOURCES = TokenBucketTest.cpp token_bucket_test_LDADD = libfollytestmain.la $(top_builddir)/libfollybenchmark.la TESTS += token_bucket_test -thread_name_test_SOURCES = ThreadNameTest.cpp +thread_name_test_SOURCES = system/ThreadNameTest.cpp thread_name_test_LDADD = libfollytestmain.la TESTS += thread_name_test diff --git a/folly/test/MemoryMappingTest.cpp b/folly/test/MemoryMappingTest.cpp deleted file mode 100644 index 958f97c3..00000000 --- a/folly/test/MemoryMappingTest.cpp +++ /dev/null @@ -1,179 +0,0 @@ -/* - * Copyright 2017 Facebook, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include - -#include -#include -#include -#include -#include - -static constexpr double kSomeDouble = 3.14; - -namespace folly { - -TEST(MemoryMapping, Basic) { - File f = File::temporary(); - { - MemoryMapping m(File(f.fd()), 0, sizeof(double), MemoryMapping::writable()); - double* d = m.asWritableRange().data(); - *d = 37 * kSomeDouble; - } - { - MemoryMapping m(File(f.fd()), 0, 3); - EXPECT_EQ(0, m.asRange().size()); // not big enough - } - { - MemoryMapping m(File(f.fd()), 0, sizeof(double)); - const double* d = m.asRange().data(); - EXPECT_EQ(*d, 37 * kSomeDouble); - } -} - -TEST(MemoryMapping, Move) { - File f = File::temporary(); - { - MemoryMapping m( - File(f.fd()), 0, sizeof(double) * 2, MemoryMapping::writable()); - double* d = m.asWritableRange().data(); - d[0] = 37 * kSomeDouble; - MemoryMapping m2(std::move(m)); - double* d2 = m2.asWritableRange().data(); - d2[1] = 39 * kSomeDouble; - } - { - MemoryMapping m(File(f.fd()), 0, sizeof(double)); - const double* d = m.asRange().data(); - EXPECT_EQ(d[0], 37 * kSomeDouble); - MemoryMapping m2(std::move(m)); - const double* d2 = m2.asRange().data(); - EXPECT_EQ(d2[1], 39 * kSomeDouble); - } -} - -TEST(MemoryMapping, DoublyMapped) { - File f = File::temporary(); - // two mappings of the same memory, different addresses. - MemoryMapping mw(File(f.fd()), 0, sizeof(double), MemoryMapping::writable()); - MemoryMapping mr(File(f.fd()), 0, sizeof(double)); - - double* dw = mw.asWritableRange().data(); - const double* dr = mr.asRange().data(); - - // Show that it's truly the same value, even though the pointers differ - EXPECT_NE(dw, dr); - *dw = 42 * kSomeDouble; - EXPECT_EQ(*dr, 42 * kSomeDouble); - *dw = 43 * kSomeDouble; - EXPECT_EQ(*dr, 43 * kSomeDouble); -} - -namespace { - -void writeStringToFileOrDie(const std::string& str, int fd) { - const char* b = str.c_str(); - size_t count = str.size(); - ssize_t total_bytes = 0; - ssize_t r; - do { - r = write(fd, b, count); - if (r == -1) { - if (errno == EINTR) { - continue; - } - PCHECK(r) << "write"; - } - - total_bytes += r; - b += r; - count -= r; - } while (r != 0 && count); -} - -} // namespace - -TEST(MemoryMapping, Simple) { - File f = File::temporary(); - writeStringToFileOrDie("hello", f.fd()); - - { - MemoryMapping m(File(f.fd())); - EXPECT_EQ("hello", m.data()); - } - { - MemoryMapping m(File(f.fd()), 1, 2); - EXPECT_EQ("el", m.data()); - } -} - -TEST(MemoryMapping, LargeFile) { - std::string fileData; - size_t fileSize = sysconf(_SC_PAGESIZE) * 3 + 10; - fileData.reserve(fileSize); - for (size_t i = 0; i < fileSize; i++) { - fileData.push_back(0xff & Random::rand32()); - } - - File f = File::temporary(); - writeStringToFileOrDie(fileData, f.fd()); - - { - MemoryMapping m(File(f.fd())); - EXPECT_EQ(fileData, m.data()); - } - { - size_t size = sysconf(_SC_PAGESIZE) * 2; - StringPiece s(fileData.data() + 9, size - 9); - MemoryMapping m(File(f.fd()), 9, size - 9); - EXPECT_EQ(s.toString(), m.data()); - } -} - -TEST(MemoryMapping, ZeroLength) { - File f = File::temporary(); - MemoryMapping m(File(f.fd())); - EXPECT_TRUE(m.mlock(MemoryMapping::LockMode::MUST_LOCK)); - EXPECT_TRUE(m.mlocked()); - EXPECT_EQ(0, m.data().size()); -} - -TEST(MemoryMapping, Advise) { - File f = File::temporary(); - size_t kPageSize = 4096; - size_t size = kPageSize + 10; // unaligned file size - PCHECK(ftruncateNoInt(f.fd(), size) == 0) << size; - - MemoryMapping m(File(f.fd())); - - // NOTE: advise crashes on bad input. - - m.advise(MADV_NORMAL, 0, kPageSize); - m.advise(MADV_NORMAL, 1, kPageSize); - m.advise(MADV_NORMAL, 0, 2); - m.advise(MADV_NORMAL, 1, 2); - - m.advise(MADV_NORMAL, kPageSize, 0); - m.advise(MADV_NORMAL, kPageSize, 1); - m.advise(MADV_NORMAL, kPageSize, size - kPageSize); - - auto off = kPageSize + 1; - m.advise(MADV_NORMAL, off, size - off); - - EXPECT_DEATH(m.advise(MADV_NORMAL, off, size - off + 1), ""); -} - -} // namespace folly diff --git a/folly/test/ShellTest.cpp b/folly/test/ShellTest.cpp deleted file mode 100644 index 697ecd8e..00000000 --- a/folly/test/ShellTest.cpp +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Copyright 2017 Facebook, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include - -#include -#include - -using namespace folly; - -TEST(Shell, ShellQuote) { - EXPECT_EQ(shellQuote("a"), "'a'"); - EXPECT_EQ(shellQuote("a'b"), "'a'\\''b'"); - EXPECT_EQ(shellQuote("a\"b"), "'a\"b'"); -} - -TEST(Shell, Shellify) { - auto command = "rm -rf /"_shellify(); - EXPECT_EQ(command[0], "/bin/sh"); - EXPECT_EQ(command[1], "-c"); - EXPECT_EQ(command[2], "rm -rf /"); - - command = "rm -rf {}"_shellify("someFile.txt"); - EXPECT_EQ(command[2], "rm -rf 'someFile.txt'"); - - command = "rm -rf {}"_shellify(5); - EXPECT_EQ(command[2], "rm -rf '5'"); - - command = "ls {}"_shellify("blah'; rm -rf /"); - EXPECT_EQ(command[2], "ls 'blah'\\''; rm -rf /'"); -} - -TEST(Shell, Shellify_deprecated) { - auto command = shellify("rm -rf /"); - EXPECT_EQ(command[0], "/bin/sh"); - EXPECT_EQ(command[1], "-c"); - EXPECT_EQ(command[2], "rm -rf /"); - - command = shellify("rm -rf {}", "someFile.txt"); - EXPECT_EQ(command[2], "rm -rf 'someFile.txt'"); - - command = shellify("rm -rf {}", 5); - EXPECT_EQ(command[2], "rm -rf '5'"); - - command = shellify("ls {}", "blah'; rm -rf /"); - EXPECT_EQ(command[2], "ls 'blah'\\''; rm -rf /'"); -} diff --git a/folly/test/ThreadCachedIntTest.cpp b/folly/test/ThreadCachedIntTest.cpp index e09c780f..6b438f74 100644 --- a/folly/test/ThreadCachedIntTest.cpp +++ b/folly/test/ThreadCachedIntTest.cpp @@ -24,9 +24,9 @@ #include #include -#include #include #include +#include using namespace folly; diff --git a/folly/test/ThreadIdTest.cpp b/folly/test/ThreadIdTest.cpp deleted file mode 100644 index 88e6fac4..00000000 --- a/folly/test/ThreadIdTest.cpp +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright 2017 Facebook, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Make sure we include ThreadId.h before anything else. -// There is no ThreadId.cpp file, so this test is the only thing that verifies -// that ThreadId.h compiles by itself when included first. -#include - -#include - -#include - -TEST(ThreadId, getCurrentID) { - auto thisThreadID = folly::getCurrentThreadID(); - uint64_t otherThreadID; - std::thread otherThread{[&] { otherThreadID = folly::getCurrentThreadID(); }}; - otherThread.join(); - EXPECT_NE(thisThreadID, otherThreadID); -} - -TEST(ThreadId, getOSThreadID) { - auto thisThreadID = folly::getOSThreadID(); - uint64_t otherThreadID; - std::thread otherThread{[&] { otherThreadID = folly::getOSThreadID(); }}; - otherThread.join(); - EXPECT_NE(thisThreadID, otherThreadID); -} diff --git a/folly/test/ThreadLocalTest.cpp b/folly/test/ThreadLocalTest.cpp index 90d9f63c..6be3f228 100644 --- a/folly/test/ThreadLocalTest.cpp +++ b/folly/test/ThreadLocalTest.cpp @@ -38,10 +38,10 @@ #include #include -#include #include #include #include +#include using namespace folly; diff --git a/folly/test/ThreadNameTest.cpp b/folly/test/ThreadNameTest.cpp deleted file mode 100644 index d47416b3..00000000 --- a/folly/test/ThreadNameTest.cpp +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Copyright 2017 Facebook, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include - -#include -#include -#include -#include - -using namespace std; -using namespace folly; - -namespace { - -const bool expectedSetOtherThreadNameResult = folly::canSetOtherThreadName(); -const bool expectedSetSelfThreadNameResult = folly::canSetCurrentThreadName(); -constexpr StringPiece kThreadName{"rockin-thread"}; - -} // namespace - -TEST(ThreadName, getCurrentThreadName) { - thread th([] { - EXPECT_EQ(expectedSetSelfThreadNameResult, setThreadName(kThreadName)); - if (expectedSetSelfThreadNameResult) { - EXPECT_EQ(kThreadName.toString(), *getCurrentThreadName()); - } - }); - SCOPE_EXIT { th.join(); }; -} - -TEST(ThreadName, setThreadName_other_pthread) { - Baton<> handle_set; - Baton<> let_thread_end; - pthread_t handle; - thread th([&] { - handle = pthread_self(); - handle_set.post(); - let_thread_end.wait(); - }); - SCOPE_EXIT { th.join(); }; - handle_set.wait(); - SCOPE_EXIT { let_thread_end.post(); }; - EXPECT_EQ( - expectedSetOtherThreadNameResult, setThreadName(handle, kThreadName)); -} - -TEST(ThreadName, setThreadName_other_id) { - Baton<> let_thread_end; - thread th([&] { - let_thread_end.wait(); - }); - SCOPE_EXIT { th.join(); }; - SCOPE_EXIT { let_thread_end.post(); }; - EXPECT_EQ( - expectedSetOtherThreadNameResult, - setThreadName(th.get_id(), kThreadName)); - if (expectedSetOtherThreadNameResult) { - EXPECT_EQ(*getThreadName(th.get_id()), kThreadName); - } -}