2 * Copyright 2012 Facebook, Inc.
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
17 #include "folly/experimental/io/HugePages.h"
21 #include <sys/types.h>
29 #include <system_error>
31 #include <boost/noncopyable.hpp>
32 #include <boost/regex.hpp>
34 #include <glog/logging.h>
36 #include "folly/Conv.h"
37 #include "folly/Format.h"
38 #include "folly/Range.h"
39 #include "folly/ScopeGuard.h"
40 #include "folly/String.h"
42 #include "folly/experimental/Gen.h"
43 #include "folly/experimental/FileGen.h"
44 #include "folly/experimental/StringGen.h"
50 // Get the default huge page size
51 size_t getDefaultHugePageSize() {
52 // We need to parse /proc/meminfo
53 static const boost::regex regex(R"!(Hugepagesize:\s*(\d+)\s*kB)!");
57 bool error = gen::byLine("/proc/meminfo") |
58 [&] (StringPiece line) -> bool {
59 if (boost::regex_match(line.begin(), line.end(), match, regex)) {
60 StringPiece numStr(line.begin() + match.position(1), match.length(1));
61 pageSize = to<size_t>(numStr) * 1024; // in KiB
68 throw std::runtime_error("Can't find default huge page size");
73 // Get raw huge page sizes (without mount points, they'll be filled later)
74 HugePageSizeVec getRawHugePageSizes() {
75 // We need to parse file names from /sys/kernel/mm/hugepages
76 static const boost::regex regex(R"!(hugepages-(\d+)kB)!");
79 fs::path path("/sys/kernel/mm/hugepages");
80 for (fs::directory_iterator it(path); it != fs::directory_iterator(); ++it) {
81 std::string filename(it->path().filename().native());
82 if (boost::regex_match(filename, match, regex)) {
83 StringPiece numStr(filename.data() + match.position(1), match.length(1));
84 vec.emplace_back(to<size_t>(numStr) * 1024);
90 // Parse the value of a pagesize mount option
91 // Format: number, optional K/M/G/T suffix, trailing junk allowed
92 size_t parsePageSizeValue(StringPiece value) {
93 static const boost::regex regex(R"!((\d+)([kmgt])?.*)!", boost::regex::icase);
95 if (!boost::regex_match(value.begin(), value.end(), match, regex)) {
96 throw std::runtime_error("Invalid pagesize option");
99 if (match.length(2) != 0) {
100 c = tolower(value[match.position(2)]);
102 StringPiece numStr(value.data() + match.position(1), match.length(1));
103 size_t size = to<size_t>(numStr);
105 case 't': size *= 1024;
106 case 'g': size *= 1024;
107 case 'm': size *= 1024;
108 case 'k': size *= 1024;
114 * Get list of supported huge page sizes and their mount points, if
115 * hugetlbfs file systems are mounted for those sizes.
117 HugePageSizeVec getHugePageSizes() {
118 HugePageSizeVec sizeVec = getRawHugePageSizes();
119 if (sizeVec.empty()) {
120 return sizeVec; // nothing to do
122 std::sort(sizeVec.begin(), sizeVec.end());
124 size_t defaultHugePageSize = getDefaultHugePageSize();
126 struct PageSizeLess {
127 bool operator()(const HugePageSize& a, size_t b) const {
130 bool operator()(size_t a, const HugePageSize& b) const {
135 // Read and parse /proc/mounts
136 std::vector<StringPiece> parts;
137 std::vector<StringPiece> options;
139 gen::byLine("/proc/mounts") | gen::eachAs<StringPiece>() |
140 [&](StringPiece line) {
142 split(" ", line, parts);
143 // device path fstype options uid gid
144 if (parts.size() != 6) {
145 throw std::runtime_error("Invalid /proc/mounts line");
147 if (parts[2] != "hugetlbfs") {
148 return; // we only care about hugetlbfs
152 split(",", parts[3], options);
153 size_t pageSize = defaultHugePageSize;
154 // Search for the "pagesize" option, which must have a value
155 for (auto& option : options) {
157 const char* p = static_cast<const char*>(
158 memchr(option.data(), '=', option.size()));
162 if (StringPiece(option.data(), p) != "pagesize") {
165 pageSize = parsePageSizeValue(StringPiece(p + 1, option.end()));
169 auto pos = std::lower_bound(sizeVec.begin(), sizeVec.end(), pageSize,
171 if (pos == sizeVec.end() || pos->size != pageSize) {
172 throw std::runtime_error("Mount page size not found");
174 if (pos->mountPoint.empty()) {
176 pos->mountPoint = fs::canonical(fs::path(parts[1].begin(),
184 // RAII wrapper around an open file, closes on exit unless you call release()
185 class ScopedFd : private boost::noncopyable {
187 explicit ScopedFd(int fd) : fd_(fd) { }
188 int fd() const { return fd_; }
198 int r = ::close(fd_);
201 throw std::system_error(errno, std::system_category(), "close failed");
209 PLOG(ERROR) << "close failed!";
217 // RAII wrapper that deletes a file upon destruction unless you call release()
218 class ScopedDeleter : private boost::noncopyable {
220 explicit ScopedDeleter(fs::path name) : name_(std::move(name)) { }
229 int r = ::unlink(name_.c_str());
231 PLOG(ERROR) << "unlink failed";
238 // RAII wrapper around a mmap mapping, munmaps upon destruction unless you
240 class ScopedMmap : private boost::noncopyable {
242 ScopedMmap(void* start, size_t size) : start_(start), size_(size) { }
244 void* start() const { return start_; }
245 size_t size() const { return size_; }
252 if (start_ == MAP_FAILED) {
255 int r = ::munmap(start_, size_);
258 throw std::system_error(errno, std::system_category(), "munmap failed");
266 PLOG(ERROR) << "munmap failed!";
276 HugePages::HugePages() : sizes_(getHugePageSizes()) { }
278 const HugePageSize& HugePages::getSize(size_t hugePageSize) const {
279 // Linear search is just fine.
280 for (auto& p : sizes_) {
281 if (p.mountPoint.empty()) {
282 continue; // not mounted
284 if (hugePageSize == 0 || hugePageSize == p.size) {
288 throw std::runtime_error("Huge page not supported / not mounted");
291 HugePages::File HugePages::create(ByteRange data,
292 const fs::path& path,
293 HugePageSize hugePageSize) const {
294 namespace bsys = ::boost::system;
295 if (hugePageSize.size == 0) {
296 hugePageSize = getSize();
301 file.size = data.size() / hugePageSize.size * hugePageSize.size;
302 if (file.size != data.size()) {
303 file.size += hugePageSize.size;
307 file.path = fs::canonical_parent(path, hugePageSize.mountPoint);
308 if (!fs::starts_with(file.path, hugePageSize.mountPoint)) {
309 throw fs::filesystem_error(
310 "HugePages::create: path not rooted at mount point",
311 file.path, hugePageSize.mountPoint,
312 bsys::errc::make_error_code(bsys::errc::invalid_argument));
315 ScopedFd fd(open(file.path.c_str(), O_RDWR | O_CREAT | O_TRUNC, 0666));
317 throw std::system_error(errno, std::system_category(), "open failed");
320 ScopedDeleter deleter(file.path);
322 ScopedMmap map(mmap(nullptr, file.size, PROT_READ | PROT_WRITE,
323 MAP_SHARED | MAP_POPULATE, fd.fd(), 0),
325 if (map.start() == MAP_FAILED) {
326 throw std::system_error(errno, std::system_category(), "mmap failed");
329 // Ignore madvise return code
330 madvise(const_cast<unsigned char*>(data.data()), data.size(),
332 // Why is this not memcpy, you ask?
333 // The SSSE3-optimized memcpy in glibc likes to copy memory backwards,
334 // rendering any prefetching from madvise useless (even harmful).
335 const unsigned char* src = data.data();
336 size_t size = data.size();
337 unsigned char* dest = reinterpret_cast<unsigned char*>(map.start());
338 if (reinterpret_cast<uintptr_t>(src) % 8 == 0) {
339 const uint64_t* src8 = reinterpret_cast<const uint64_t*>(src);
340 size_t size8 = size / 8;
341 uint64_t* dest8 = reinterpret_cast<uint64_t*>(dest);
345 src = reinterpret_cast<const unsigned char*>(src8);
346 dest = reinterpret_cast<unsigned char*>(dest8);
349 memcpy(dest, src, size);