2 * Copyright 2014 Facebook, Inc.
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
17 #include "MemoryIdler.h"
18 #include <folly/Logging.h>
19 #include <folly/Malloc.h>
20 #include <folly/ScopeGuard.h>
21 #include <folly/detail/CacheLocality.h>
31 // weak linking means the symbol will be null if not available, instead
33 extern "C" int mallctl(const char *name, void *oldp, size_t *oldlenp,
34 void *newp, size_t newlen)
35 __attribute__((weak));
38 namespace folly { namespace detail {
40 AtomicStruct<std::chrono::steady_clock::duration>
41 MemoryIdler::defaultIdleTimeout(std::chrono::seconds(5));
44 /// Calls mallctl, optionally reading and/or writing an unsigned value
45 /// if in and/or out is non-null. Logs on error
46 static unsigned mallctlWrapper(const char* cmd, const unsigned* in,
48 size_t outLen = sizeof(unsigned);
49 int err = mallctl(cmd,
50 out, out ? &outLen : nullptr,
51 const_cast<unsigned*>(in), in ? sizeof(unsigned) : 0);
53 FB_LOG_EVERY_MS(WARNING, 10000)
54 << "mallctl " << cmd << ": " << strerror(err) << " (" << err << ")";
59 void MemoryIdler::flushLocalMallocCaches() {
60 if (usingJEMalloc()) {
62 FB_LOG_EVERY_MS(ERROR, 10000) << "mallctl weak link failed";
66 // "tcache.flush" was renamed to "thread.tcache.flush" in jemalloc 3
67 (void)mallctlWrapper("thread.tcache.flush", nullptr, nullptr);
69 // By default jemalloc has 4 arenas per cpu, and then assigns each
70 // thread to one of those arenas. This means that in any service
71 // that doesn't perform a lot of context switching, the chances that
72 // another thread will be using the current thread's arena (and hence
73 // doing the appropriate dirty-page purging) are low. Some good
74 // tuned configurations (such as that used by hhvm) use fewer arenas
75 // and then pin threads to avoid contended access. In that case,
76 // purging the arenas is counter-productive. We use the heuristic
77 // that if narenas <= 2 * num_cpus then we shouldn't do anything here,
78 // which detects when the narenas has been reduced from the default
80 unsigned arenaForCurrent;
81 if (mallctlWrapper("arenas.narenas", nullptr, &narenas) == 0 &&
82 narenas > 2 * CacheLocality::system().numCpus &&
83 mallctlWrapper("thread.arena", nullptr, &arenaForCurrent) == 0) {
84 (void)mallctlWrapper("arenas.purge", &arenaForCurrent, nullptr);
92 static const size_t s_pageSize = sysconf(_SC_PAGESIZE);
93 static FOLLY_TLS uintptr_t tls_stackLimit;
94 static FOLLY_TLS size_t tls_stackSize;
96 static void fetchStackLimits() {
98 #if defined(_GNU_SOURCE) && defined(__linux__) // Linux+GNU extension
99 pthread_getattr_np(pthread_self(), &attr);
101 pthread_attr_init(&attr);
103 SCOPE_EXIT { pthread_attr_destroy(&attr); };
108 if ((err = pthread_attr_getstack(&attr, &addr, &rawSize))) {
109 // unexpected, but it is better to continue in prod than do nothing
110 FB_LOG_EVERY_MS(ERROR, 10000) << "pthread_attr_getstack error " << err;
115 assert(addr != nullptr);
116 assert(rawSize >= PTHREAD_STACK_MIN);
118 // glibc subtracts guard page from stack size, even though pthread docs
119 // seem to imply the opposite
121 if (pthread_attr_getguardsize(&attr, &guardSize) != 0) {
124 assert(rawSize > guardSize);
126 // stack goes down, so guard page adds to the base addr
127 tls_stackLimit = uintptr_t(addr) + guardSize;
128 tls_stackSize = rawSize - guardSize;
130 assert((tls_stackLimit & (s_pageSize - 1)) == 0);
133 FOLLY_NOINLINE static uintptr_t getStackPtr() {
135 auto rv = uintptr_t(&marker);
139 void MemoryIdler::unmapUnusedStack(size_t retain) {
140 if (tls_stackSize == 0) {
143 if (tls_stackSize <= std::max(size_t(1), retain)) {
144 // covers both missing stack info, and impossibly large retain
148 auto sp = getStackPtr();
149 assert(sp >= tls_stackLimit);
150 assert(sp - tls_stackLimit < tls_stackSize);
152 auto end = (sp - retain) & ~(s_pageSize - 1);
153 if (end <= tls_stackLimit) {
154 // no pages are eligible for unmapping
158 size_t len = end - tls_stackLimit;
159 assert((len & (s_pageSize - 1)) == 0);
160 if (madvise((void*)tls_stackLimit, len, MADV_DONTNEED) != 0) {
161 // It is likely that the stack vma hasn't been fully grown. In this
162 // case madvise will apply dontneed to the present vmas, then return
163 // errno of ENOMEM. We can also get an EAGAIN, theoretically.
164 // EINVAL means either an invalid alignment or length, or that some
165 // of the pages are locked or shared. Neither should occur.
166 assert(errno == EAGAIN || errno == ENOMEM);
172 void MemoryIdler::unmapUnusedStack(size_t retain) {