2 * Copyright 2016 Facebook, Inc.
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
17 #include <folly/FileUtil.h>
18 #include <folly/detail/FileUtilDetail.h>
19 #include <folly/experimental/TestUtil.h>
22 #if defined(__linux__)
26 #include <glog/logging.h>
28 #include <folly/Exception.h>
29 #include <folly/File.h>
30 #include <folly/Range.h>
31 #include <folly/String.h>
32 #include <folly/portability/GTest.h>
34 namespace folly { namespace test {
36 using namespace fileutil_detail;
43 Reader(off_t offset, StringPiece data, std::deque<ssize_t> spec);
46 ssize_t operator()(int fd, void* buf, size_t count);
49 ssize_t operator()(int fd, void* buf, size_t count, off_t offset);
52 ssize_t operator()(int fd, const iovec* iov, int count);
55 ssize_t operator()(int fd, const iovec* iov, int count, off_t offset);
57 const std::deque<ssize_t> spec() const { return spec_; }
64 std::deque<ssize_t> spec_;
67 Reader::Reader(off_t offset, StringPiece data, std::deque<ssize_t> spec)
70 spec_(std::move(spec)) {
73 ssize_t Reader::nextSize() {
75 throw std::runtime_error("spec empty");
77 ssize_t n = spec_.front();
83 spec_.clear(); // so we fail if called again
90 ssize_t Reader::operator()(int /* fd */, void* buf, size_t count) {
91 ssize_t n = nextSize();
95 if (size_t(n) > count) {
96 throw std::runtime_error("requested count too small");
98 memcpy(buf, data_.data(), n);
103 ssize_t Reader::operator()(int fd, void* buf, size_t count, off_t offset) {
104 EXPECT_EQ(offset_, offset);
105 return operator()(fd, buf, count);
108 ssize_t Reader::operator()(int /* fd */, const iovec* iov, int count) {
109 ssize_t n = nextSize();
113 ssize_t remaining = n;
114 for (; count != 0 && remaining != 0; ++iov, --count) {
115 ssize_t len = std::min(remaining, ssize_t(iov->iov_len));
116 memcpy(iov->iov_base, data_.data(), len);
120 if (remaining != 0) {
121 throw std::runtime_error("requested total size too small");
126 ssize_t Reader::operator()(int fd, const iovec* iov, int count, off_t offset) {
127 EXPECT_EQ(offset_, offset);
128 return operator()(fd, iov, count);
133 class FileUtilTest : public ::testing::Test {
137 Reader reader(std::deque<ssize_t> spec);
140 std::vector<std::pair<size_t, Reader>> readers_;
143 FileUtilTest::FileUtilTest()
144 : in_("1234567890abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ") {
145 CHECK_EQ(62, in_.size());
147 readers_.emplace_back(0, reader({0}));
148 readers_.emplace_back(62, reader({62}));
149 readers_.emplace_back(62, reader({62, -1})); // error after end (not called)
150 readers_.emplace_back(61, reader({61, 0}));
151 readers_.emplace_back(-1, reader({61, -1})); // error before end
152 readers_.emplace_back(62, reader({31, 31}));
153 readers_.emplace_back(62, reader({1, 10, 20, 10, 1, 20}));
154 readers_.emplace_back(61, reader({1, 10, 20, 10, 20, 0}));
155 readers_.emplace_back(41, reader({1, 10, 20, 10, 0}));
156 readers_.emplace_back(-1, reader({1, 10, 20, 10, 20, -1}));
159 Reader FileUtilTest::reader(std::deque<ssize_t> spec) {
160 return Reader(42, in_, std::move(spec));
163 TEST_F(FileUtilTest, read) {
164 for (auto& p : readers_) {
165 std::string out(in_.size(), '\0');
166 EXPECT_EQ(p.first, wrapFull(p.second, 0, &out[0], out.size()));
167 if (p.first != (decltype(p.first))(-1)) {
168 EXPECT_EQ(in_.substr(0, p.first), out.substr(0, p.first));
173 TEST_F(FileUtilTest, pread) {
174 for (auto& p : readers_) {
175 std::string out(in_.size(), '\0');
176 EXPECT_EQ(p.first, wrapFull(p.second, 0, &out[0], out.size(), off_t(42)));
177 if (p.first != (decltype(p.first))(-1)) {
178 EXPECT_EQ(in_.substr(0, p.first), out.substr(0, p.first));
185 explicit IovecBuffers(std::initializer_list<size_t> sizes);
186 explicit IovecBuffers(std::vector<size_t> sizes);
188 std::vector<iovec> iov() const { return iov_; } // yes, make a copy
189 std::string join() const { return folly::join("", buffers_); }
193 std::vector<std::string> buffers_;
194 std::vector<iovec> iov_;
197 IovecBuffers::IovecBuffers(std::initializer_list<size_t> sizes) {
198 iov_.reserve(sizes.size());
199 for (auto& s : sizes) {
200 buffers_.push_back(std::string(s, '\0'));
202 for (auto& b : buffers_) {
204 iov.iov_base = &b[0];
205 iov.iov_len = b.size();
210 IovecBuffers::IovecBuffers(std::vector<size_t> sizes) {
211 iov_.reserve(sizes.size());
212 for (auto s : sizes) {
213 buffers_.push_back(std::string(s, '\0'));
215 for (auto& b : buffers_) {
217 iov.iov_base = &b[0];
218 iov.iov_len = b.size();
223 size_t IovecBuffers::size() const {
225 for (auto& b : buffers_) {
231 TEST_F(FileUtilTest, readv) {
232 for (auto& p : readers_) {
233 IovecBuffers buf({12, 19, 31});
234 ASSERT_EQ(62, buf.size());
236 auto iov = buf.iov();
237 EXPECT_EQ(p.first, wrapvFull(p.second, 0, iov.data(), iov.size()));
238 if (p.first != (decltype(p.first))(-1)) {
239 EXPECT_EQ(in_.substr(0, p.first), buf.join().substr(0, p.first));
244 TEST(FileUtilTest2, wrapv) {
245 TemporaryFile tempFile("file-util-test");
246 std::vector<size_t> sizes;
248 for (int32_t i = 0; i < 1500; ++i) {
249 sizes.push_back(i % 3 + 1);
252 IovecBuffers buf(sizes);
253 ASSERT_EQ(sum, buf.size());
254 auto iov = buf.iov();
255 EXPECT_EQ(sum, wrapvFull(writev, tempFile.fd(), iov.data(), iov.size()));
258 TEST_F(FileUtilTest, preadv) {
259 for (auto& p : readers_) {
260 IovecBuffers buf({12, 19, 31});
261 ASSERT_EQ(62, buf.size());
263 auto iov = buf.iov();
265 wrapvFull(p.second, 0, iov.data(), iov.size(), off_t(42)));
266 if (p.first != (decltype(p.first))(-1)) {
267 EXPECT_EQ(in_.substr(0, p.first), buf.join().substr(0, p.first));
272 TEST(String, readFile) {
273 const TemporaryFile afileTemp, emptyFileTemp;
274 auto afile = afileTemp.path().string();
275 auto emptyFile = emptyFileTemp.path().string();
277 EXPECT_TRUE(writeFile(string(), emptyFile.c_str()));
278 EXPECT_TRUE(writeFile(StringPiece("bar"), afile.c_str()));
282 EXPECT_TRUE(readFile(emptyFile.c_str(), contents));
283 EXPECT_EQ(contents, "");
284 EXPECT_TRUE(readFile(afile.c_str(), contents, 0));
285 EXPECT_EQ("", contents);
286 EXPECT_TRUE(readFile(afile.c_str(), contents, 2));
287 EXPECT_EQ("ba", contents);
288 EXPECT_TRUE(readFile(afile.c_str(), contents));
289 EXPECT_EQ("bar", contents);
292 vector<unsigned char> contents;
293 EXPECT_TRUE(readFile(emptyFile.c_str(), contents));
294 EXPECT_EQ(vector<unsigned char>(), contents);
295 EXPECT_TRUE(readFile(afile.c_str(), contents, 0));
296 EXPECT_EQ(vector<unsigned char>(), contents);
297 EXPECT_TRUE(readFile(afile.c_str(), contents, 2));
298 EXPECT_EQ(vector<unsigned char>({'b', 'a'}), contents);
299 EXPECT_TRUE(readFile(afile.c_str(), contents));
300 EXPECT_EQ(vector<unsigned char>({'b', 'a', 'r'}), contents);
304 class ReadFileFd : public ::testing::Test {
306 void SetUp() override {
307 ASSERT_TRUE(writeFile(StringPiece("bar"), aFile.path().string().c_str()));
313 TEST_F(ReadFileFd, ReadZeroBytes) {
314 std::string contents;
315 EXPECT_TRUE(readFile(aFile.fd(), contents, 0));
316 EXPECT_EQ("", contents);
319 TEST_F(ReadFileFd, ReadPartial) {
320 std::string contents;
321 EXPECT_TRUE(readFile(aFile.fd(), contents, 2));
322 EXPECT_EQ("ba", contents);
325 TEST_F(ReadFileFd, ReadFull) {
326 std::string contents;
327 EXPECT_TRUE(readFile(aFile.fd(), contents));
328 EXPECT_EQ("bar", contents);
331 TEST_F(ReadFileFd, WriteOnlyFd) {
332 File f(aFile.path().string(), O_WRONLY);
333 std::string contents;
334 EXPECT_FALSE(readFile(f.fd(), contents));
338 TEST_F(ReadFileFd, InvalidFd) {
339 File f(aFile.path().string());
341 std::string contents;
342 msvcSuppressAbortOnInvalidParams([&] {
343 EXPECT_FALSE(readFile(f.fd(), contents));
348 class WriteFileAtomic : public ::testing::Test {
352 std::set<std::string> listTmpDir() const {
353 std::set<std::string> entries;
354 for (auto& entry : fs::directory_iterator(tmpDir_.path())) {
355 entries.insert(entry.path().filename().string());
360 std::string readData(const string& path) const {
362 if (!readFile(path.c_str(), data)) {
363 throwSystemError("failed to read ", path);
368 struct stat statFile(const string& path) const {
370 auto rc = stat(path.c_str(), &s);
371 checkUnixError(rc, "failed to stat() ", path);
375 mode_t getPerms(const string& path) {
376 return (statFile(path).st_mode & 0777);
379 string tmpPath(StringPiece name) {
380 return tmpDir_.path().string() + "/" + name.str();
383 void setDirPerms(mode_t mode) {
384 auto rc = chmod(tmpDir_.path().string().c_str(), mode);
385 checkUnixError(rc, "failed to set permissions on tmp dir");
388 TemporaryDirectory tmpDir_{"folly_file_test"};
391 TEST_F(WriteFileAtomic, writeNew) {
392 // Call writeFileAtomic() to create a new file
393 auto path = tmpPath("foo");
394 auto contents = StringPiece{"contents\n"};
395 writeFileAtomic(path, contents);
397 // The directory should contain exactly 1 file now, with the correct contents
398 EXPECT_EQ(set<string>{"foo"}, listTmpDir());
399 EXPECT_EQ(contents, readData(path));
400 EXPECT_EQ(0644, getPerms(path));
403 TEST_F(WriteFileAtomic, overwrite) {
404 // Call writeFileAtomic() to create a new file
405 auto path = tmpPath("foo");
406 auto contents1 = StringPiece{"contents\n"};
407 writeFileAtomic(path, contents1);
409 EXPECT_EQ(set<string>{"foo"}, listTmpDir());
410 EXPECT_EQ(contents1, readData(path));
411 EXPECT_EQ(0644, getPerms(path));
413 // Now overwrite the file with different contents
414 auto contents2 = StringPiece{"testing"};
415 writeFileAtomic(path, contents2);
416 EXPECT_EQ(set<string>{"foo"}, listTmpDir());
417 EXPECT_EQ(contents2, readData(path));
418 EXPECT_EQ(0644, getPerms(path));
420 // Test overwriting with relatively large contents, and different permissions
422 "asdf" + string(10240, '\n') + "foobar\n" + string(10240, 'b') + "\n";
423 writeFileAtomic(path, contents3, 0444);
424 EXPECT_EQ(set<string>{"foo"}, listTmpDir());
425 EXPECT_EQ(contents3, readData(path));
426 EXPECT_EQ(0444, getPerms(path));
428 // Test overwriting with empty contents
430 // Note that the file's permissions are 0444 at this point (read-only),
431 // but we writeFileAtomic() should still replace it successfully. Since we
432 // update it with a rename we need write permissions on the parent directory,
433 // but not the destination file.
434 auto contents4 = StringPiece("");
435 writeFileAtomic(path, contents4, 0400);
436 EXPECT_EQ(set<string>{"foo"}, listTmpDir());
437 EXPECT_EQ(contents4, readData(path));
438 EXPECT_EQ(0400, getPerms(path));
441 TEST_F(WriteFileAtomic, directoryPermissions) {
442 // Test writeFileAtomic() when we do not have write permission in the target
445 // Make the test directory read-only
448 // Restore directory permissions before we exit, just to ensure the code
449 // will be able to clean up the directory.
452 } catch (const std::exception&) {
453 // Intentionally ignore errors here, in case an exception is already
458 // writeFileAtomic() should fail, and the directory should still be empty
459 auto path1 = tmpPath("foo");
460 auto contents = StringPiece("testing");
461 EXPECT_THROW(writeFileAtomic(path1, contents), std::system_error);
462 EXPECT_EQ(set<string>{}, listTmpDir());
464 // Make the directory writable again, then create the file
466 writeFileAtomic(path1, contents, 0400);
467 EXPECT_EQ(contents, readData(path1));
468 EXPECT_EQ(set<string>{"foo"}, listTmpDir());
470 // Make the directory read-only again
471 // Creating another file now should fail and we should still have only the
475 writeFileAtomic(tmpPath("another_file.txt"), "x\n"), std::system_error);
476 EXPECT_EQ(set<string>{"foo"}, listTmpDir());
479 TEST_F(WriteFileAtomic, multipleFiles) {
480 // Test creating multiple files in the same directory
481 writeFileAtomic(tmpPath("foo.txt"), "foo");
482 writeFileAtomic(tmpPath("bar.txt"), "bar", 0400);
483 writeFileAtomic(tmpPath("foo_txt"), "underscore", 0440);
484 writeFileAtomic(tmpPath("foo.txt2"), "foo2", 0444);
486 auto expectedPaths = set<string>{"foo.txt", "bar.txt", "foo_txt", "foo.txt2"};
487 EXPECT_EQ(expectedPaths, listTmpDir());
488 EXPECT_EQ("foo", readData(tmpPath("foo.txt")));
489 EXPECT_EQ("bar", readData(tmpPath("bar.txt")));
490 EXPECT_EQ("underscore", readData(tmpPath("foo_txt")));
491 EXPECT_EQ("foo2", readData(tmpPath("foo.txt2")));
492 EXPECT_EQ(0644, getPerms(tmpPath("foo.txt")));
493 EXPECT_EQ(0400, getPerms(tmpPath("bar.txt")));
494 EXPECT_EQ(0440, getPerms(tmpPath("foo_txt")));
495 EXPECT_EQ(0444, getPerms(tmpPath("foo.txt2")));
499 #if defined(__linux__)
502 * A helper class that forces our fchmod() wrapper to fail when
503 * an FChmodFailure object exists.
505 class FChmodFailure {
514 static bool shouldFail() {
515 return forceFailure_.load() > 0;
519 static std::atomic<int> forceFailure_;
522 std::atomic<int> FChmodFailure::forceFailure_{0};
525 // Replace the system fchmod() function with our own stub, so we can
526 // trigger failures in the writeFileAtomic() tests.
527 int fchmod(int fd, mode_t mode) {
528 static const auto realFunction =
529 reinterpret_cast<int (*)(int, mode_t)>(dlsym(RTLD_NEXT, "fchmod"));
530 // For sanity, make sure we didn't find ourself,
531 // since that would cause infinite recursion.
532 CHECK_NE(realFunction, fchmod);
534 if (FChmodFailure::shouldFail()) {
538 return realFunction(fd, mode);
543 TEST_F(WriteFileAtomic, chmodFailure) {
544 auto path = tmpPath("foo");
546 // Use our stubbed out fchmod() function to force a failure when setting up
547 // the temporary file.
549 // First try when creating the file for the first time.
552 EXPECT_THROW(writeFileAtomic(path, "foobar"), std::system_error);
554 EXPECT_EQ(set<string>{}, listTmpDir());
556 // Now create a file normally so we can overwrite it
557 auto contents = StringPiece("regular perms");
558 writeFileAtomic(path, contents, 0600);
559 EXPECT_EQ(contents, readData(path));
560 EXPECT_EQ(0600, getPerms(path));
561 EXPECT_EQ(set<string>{"foo"}, listTmpDir());
563 // Now try overwriting the file when forcing fchmod to fail
566 EXPECT_THROW(writeFileAtomic(path, "overwrite"), std::system_error);
568 // The file should be unchanged
569 EXPECT_EQ(contents, readData(path));
570 EXPECT_EQ(0600, getPerms(path));
571 EXPECT_EQ(set<string>{"foo"}, listTmpDir());