/*
- * Copyright 2013 Facebook, Inc.
+ * Copyright 2014 Facebook, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
/*
- * Copyright 2013 Facebook, Inc.
+ * Copyright 2014 Facebook, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
#define FOLLY_FILEUTIL_H_
#include "folly/Portability.h"
+#include "folly/ScopeGuard.h"
+#include <cassert>
+#include <limits>
#include <sys/stat.h>
#include <sys/types.h>
#include <sys/uio.h>
ssize_t pwritevFull(int fd, iovec* iov, int count, off_t offset);
#endif
+/**
+ * Read entire file (if num_bytes is defaulted) or no more than
+ * num_bytes (otherwise) into container *out. The container is assumed
+ * to be contiguous, with element size equal to 1, and offer size(),
+ * reserve(), and random access (e.g. std::vector<char>, std::string,
+ * fbstring).
+ *
+ * Returns: true on success or false on failure. In the latter case
+ * errno will be set appropriately by the failing system primitive.
+ */
+template <class Container>
+bool readFile(const char* file_name, Container& out,
+ size_t num_bytes = std::numeric_limits<size_t>::max()) {
+ static_assert(sizeof(out[0]) == 1,
+ "readFile: only containers with byte-sized elements accepted");
+ assert(file_name);
+
+ const auto fd = open(file_name, O_RDONLY);
+ if (fd == -1) return false;
+
+ size_t soFar = 0; // amount of bytes successfully read
+ SCOPE_EXIT {
+ assert(out.size() >= soFar); // resize better doesn't throw
+ out.resize(soFar);
+ // Ignore errors when closing the file
+ close(fd);
+ };
+
+ // Obtain file size:
+ struct stat buf;
+ if (fstat(fd, &buf) == -1) return false;
+ // Some files (notably under /proc and /sys on Linux) lie about
+ // their size, so treat the size advertised by fstat under advise
+ // but don't rely on it. In particular, if the size is zero, we
+ // should attempt to read stuff. If not zero, we'll attempt to read
+ // one extra byte.
+ constexpr size_t initialAlloc = 1024 * 4;
+ out.resize(
+ std::min(
+ buf.st_size ? buf.st_size + 1 : initialAlloc,
+ num_bytes));
+
+ while (soFar < out.size()) {
+ const auto actual = readFull(fd, &out[soFar], out.size() - soFar);
+ if (actual == -1) {
+ return false;
+ }
+ soFar += actual;
+ if (soFar < out.size()) {
+ // File exhausted
+ break;
+ }
+ // Ew, allocate more memory. Use exponential growth to avoid
+ // quadratic behavior. Cap size to num_bytes.
+ out.resize(std::min(out.size() * 3 / 2, num_bytes));
+ }
+
+ return true;
+}
+
} // namespaces
#endif /* FOLLY_FILEUTIL_H_ */
-
/*
- * Copyright 2013 Facebook, Inc.
+ * Copyright 2014 Facebook, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
} // namespace folly
#endif /* FOLLY_STRING_INL_H_ */
-
/*
- * Copyright 2013 Facebook, Inc.
+ * Copyright 2014 Facebook, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
/*
- * Copyright 2013 Facebook, Inc.
+ * Copyright 2014 Facebook, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
| resplit(delim);
}
+/**
+ * Ditto, take the filename and opens it
+ */
+inline auto byLine(const char* fileName, char delim = '\n')
+ -> decltype(byLine(File(fileName))) {
+ return byLine(File(fileName), delim);
+}
+
}} // !folly::gen
/*
- * Copyright 2013 Facebook, Inc.
+ * Copyright 2014 Facebook, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
}
} // namespace folly
-
/*
- * Copyright 2013 Facebook, Inc.
+ * Copyright 2014 Facebook, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
PCHECK(::read(rfd, &buf, 1) == 1); // wait for startup
}
- auto s = byLine(rfd) | eachTo<int64_t>() | sum;
+ auto s = byLine(File(rfd)) | eachTo<int64_t>() | sum;
folly::doNotOptimizeAway(s);
BENCHMARK_SUSPEND {
/*
- * Copyright 2013 Facebook, Inc.
+ * Copyright 2014 Facebook, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
auto collect = eachTo<std::string>() | as<vector>();
auto expected = src | resplit('\n') | collect;
- src | eachAs<StringPiece>() | toFile(file.fd(), bufferSize);
+ src | eachAs<StringPiece>() | toFile(File(file.fd()), bufferSize);
auto found = byLine(file.path().c_str()) | collect;
EXPECT_TRUE(expected == found);
/*
- * Copyright 2013 Facebook, Inc.
+ * Copyright 2014 Facebook, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
TEST(RecordIOTest, Simple) {
TemporaryFile file;
{
- RecordIOWriter writer(file.fd());
+ RecordIOWriter writer(File(file.fd()));
writer.write(iobufs({"hello ", "world"}));
writer.write(iobufs({"goodbye"}));
}
{
- RecordIOReader reader(file.fd());
+ RecordIOReader reader(File(file.fd()));
auto it = reader.begin();
ASSERT_FALSE(it == reader.end());
EXPECT_EQ("hello world", sp((it++)->first));
EXPECT_TRUE(it == reader.end());
}
{
- RecordIOWriter writer(file.fd());
+ RecordIOWriter writer(File(file.fd()));
writer.write(iobufs({"meow"}));
writer.write(iobufs({"woof"}));
}
{
- RecordIOReader reader(file.fd());
+ RecordIOReader reader(File(file.fd()));
auto it = reader.begin();
ASSERT_FALSE(it == reader.end());
EXPECT_EQ("hello world", sp((it++)->first));
memset(tmp, 'x', kSize);
TemporaryFile file;
{
- RecordIOWriter writer(file.fd());
+ RecordIOWriter writer(File(file.fd()));
for (int i = 0; i < kSize; ++i) { // record of size 0 should be ignored
writer.write(IOBuf::wrapBuffer(tmp, i));
}
}
{
- RecordIOReader reader(file.fd());
+ RecordIOReader reader(File(file.fd()));
auto it = reader.begin();
for (int i = 1; i < kSize; ++i) {
ASSERT_FALSE(it == reader.end());
TEST(RecordIOTest, MultipleFileIds) {
TemporaryFile file;
{
- RecordIOWriter writer(file.fd(), 1);
+ RecordIOWriter writer(File(file.fd()), 1);
writer.write(iobufs({"hello"}));
}
{
- RecordIOWriter writer(file.fd(), 2);
+ RecordIOWriter writer(File(file.fd()), 2);
writer.write(iobufs({"world"}));
}
{
- RecordIOWriter writer(file.fd(), 1);
+ RecordIOWriter writer(File(file.fd()), 1);
writer.write(iobufs({"goodbye"}));
}
{
- RecordIOReader reader(file.fd(), 0); // return all
+ RecordIOReader reader(File(file.fd()), 0); // return all
auto it = reader.begin();
ASSERT_FALSE(it == reader.end());
EXPECT_EQ("hello", sp((it++)->first));
EXPECT_TRUE(it == reader.end());
}
{
- RecordIOReader reader(file.fd(), 1);
+ RecordIOReader reader(File(file.fd()), 1);
auto it = reader.begin();
ASSERT_FALSE(it == reader.end());
EXPECT_EQ("hello", sp((it++)->first));
EXPECT_TRUE(it == reader.end());
}
{
- RecordIOReader reader(file.fd(), 2);
+ RecordIOReader reader(File(file.fd()), 2);
auto it = reader.begin();
ASSERT_FALSE(it == reader.end());
EXPECT_EQ("world", sp((it++)->first));
EXPECT_TRUE(it == reader.end());
}
{
- RecordIOReader reader(file.fd(), 3);
+ RecordIOReader reader(File(file.fd()), 3);
auto it = reader.begin();
EXPECT_TRUE(it == reader.end());
}
TEST(RecordIOTest, ExtraMagic) {
TemporaryFile file;
{
- RecordIOWriter writer(file.fd());
+ RecordIOWriter writer(File(file.fd()));
writer.write(iobufs({"hello"}));
}
uint8_t buf[recordio_helpers::headerSize() + 5];
// and an extra record
EXPECT_EQ(sizeof(buf), write(file.fd(), buf, sizeof(buf)));
{
- RecordIOReader reader(file.fd());
+ RecordIOReader reader(File(file.fd()));
auto it = reader.begin();
ASSERT_FALSE(it == reader.end());
EXPECT_EQ("hello", sp((it++)->first));
// Recreate the writer multiple times so we test that we create a
// continuous stream
for (size_t i = 0; i < 3; ++i) {
- RecordIOWriter writer(file.fd());
+ RecordIOWriter writer(File(file.fd()));
for (size_t j = 0; j < recordCount; ++j) {
off_t beginPos = writer.filePos();
record.clear();
{
size_t i = 0;
- RecordIOReader reader(file.fd());
+ RecordIOReader reader(File(file.fd()));
for (auto& r : reader) {
SCOPED_TRACE(i);
ASSERT_LT(i, records.size());
google::ParseCommandLineFlags(&argc, &argv, true);
return RUN_ALL_TESTS();
}
-
/*
- * Copyright 2013 Facebook, Inc.
+ * Copyright 2014 Facebook, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
namespace folly { namespace test {
using namespace fileutil_detail;
+using namespace std;
namespace {
}
#endif
+TEST(String, readFile) {
+ srand(time(nullptr));
+ const string tmpPrefix = to<string>("/tmp/folly-file-util-test-",
+ getpid(), "-", rand(), "-");
+ const string afile = tmpPrefix + "myfile";
+ const string emptyFile = tmpPrefix + "myfile2";
+
+ SCOPE_EXIT {
+ unlink(afile.c_str());
+ unlink(emptyFile.c_str());
+ };
+
+ auto f = fopen(emptyFile.c_str(), "wb");
+ EXPECT_NE(nullptr, f);
+ EXPECT_EQ(0, fclose(f));
+ f = fopen(afile.c_str(), "wb");
+ EXPECT_NE(nullptr, f);
+ EXPECT_EQ(3, fwrite("bar", 1, 3, f));
+ EXPECT_EQ(0, fclose(f));
+
+ {
+ string contents;
+ EXPECT_TRUE(readFile(emptyFile.c_str(), contents));
+ EXPECT_EQ(contents, "");
+ EXPECT_TRUE(readFile(afile.c_str(), contents, 0));
+ EXPECT_EQ("", contents);
+ EXPECT_TRUE(readFile(afile.c_str(), contents, 2));
+ EXPECT_EQ("ba", contents);
+ EXPECT_TRUE(readFile(afile.c_str(), contents));
+ EXPECT_EQ("bar", contents);
+ }
+ {
+ vector<unsigned char> contents;
+ EXPECT_TRUE(readFile(emptyFile.c_str(), contents));
+ EXPECT_EQ(vector<unsigned char>(), contents);
+ EXPECT_TRUE(readFile(afile.c_str(), contents, 0));
+ EXPECT_EQ(vector<unsigned char>(), contents);
+ EXPECT_TRUE(readFile(afile.c_str(), contents, 2));
+ EXPECT_EQ(vector<unsigned char>({'b', 'a'}), contents);
+ EXPECT_TRUE(readFile(afile.c_str(), contents));
+ EXPECT_EQ(vector<unsigned char>({'b', 'a', 'r'}), contents);
+ }
+}
}} // namespaces
google::ParseCommandLineFlags(&argc, &argv, true);
return RUN_ALL_TESTS();
}
-
/*
- * Copyright 2013 Facebook, Inc.
+ * Copyright 2014 Facebook, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
TEST(MemoryMapping, Basic) {
File f = File::temporary();
{
- WritableMemoryMapping m(f.fd(), 0, sizeof(double));
+ WritableMemoryMapping m(File(f.fd()), 0, sizeof(double));
double volatile* d = m.asWritableRange<double>().data();
*d = 37 * M_PI;
}
{
- MemoryMapping m(f.fd(), 0, 3);
+ MemoryMapping m(File(f.fd()), 0, 3);
EXPECT_EQ(0, m.asRange<int>().size()); //not big enough
}
{
- MemoryMapping m(f.fd(), 0, sizeof(double));
+ MemoryMapping m(File(f.fd()), 0, sizeof(double));
const double volatile* d = m.asRange<double>().data();
EXPECT_EQ(*d, 37 * M_PI);
}
TEST(MemoryMapping, DoublyMapped) {
File f = File::temporary();
// two mappings of the same memory, different addresses.
- WritableMemoryMapping mw(f.fd(), 0, sizeof(double));
- MemoryMapping mr(f.fd(), 0, sizeof(double));
+ WritableMemoryMapping mw(File(f.fd()), 0, sizeof(double));
+ MemoryMapping mr(File(f.fd()), 0, sizeof(double));
double volatile* dw = mw.asWritableRange<double>().data();
const double volatile* dr = mr.asRange<double>().data();
writeStringToFileOrDie("hello", f.fd());
{
- MemoryMapping m(f.fd());
+ MemoryMapping m(File(f.fd()));
EXPECT_EQ("hello", m.data());
}
{
- MemoryMapping m(f.fd(), 1, 2);
+ MemoryMapping m(File(f.fd()), 1, 2);
EXPECT_EQ("el", m.data());
}
}
writeStringToFileOrDie(fileData, f.fd());
{
- MemoryMapping m(f.fd());
+ MemoryMapping m(File(f.fd()));
EXPECT_EQ(fileData, m.data());
}
{
size_t size = sysconf(_SC_PAGESIZE) * 2;
StringPiece s(fileData.data() + 9, size - 9);
- MemoryMapping m(f.fd(), 9, size - 9);
+ MemoryMapping m(File(f.fd()), 9, size - 9);
EXPECT_EQ(s.toString(), m.data());
}
}
TEST(MemoryMapping, ZeroLength) {
File f = File::temporary();
- MemoryMapping m(f.fd());
+ MemoryMapping m(File(f.fd()));
EXPECT_TRUE(m.mlock(MemoryMapping::LockMode::MUST_LOCK));
EXPECT_TRUE(m.mlocked());
EXPECT_EQ(0, m.data().size());
/*
- * Copyright 2013 Facebook, Inc.
+ * Copyright 2014 Facebook, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
}
return ret;
}
-
/*
- * Copyright 2013 Facebook, Inc.
+ * Copyright 2014 Facebook, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
TEST(PopenSubprocessTest, PopenRead) {
Subprocess proc("ls /", Subprocess::pipeStdout());
int found = 0;
- gen::byLine(proc.stdout()) |
+ gen::byLine(File(proc.stdout())) |
[&] (StringPiece line) {
if (line == "etc" || line == "bin" || line == "usr") {
++found;
projects / folly.git / commitdiff