--- /dev/null
+/*
+ * Copyright 2012 Facebook, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef FOLLY_FILE_H_
+#error This file may only be included from folly/experimental/File.h
+#endif
+
+#include <algorithm>
+
+namespace folly {
+
+inline File::File(int fd, bool ownsFd) : fd_(fd), ownsFd_(ownsFd) { }
+
+inline File::~File() {
+ closeNoThrow(); // ignore error
+}
+
+inline void File::release() {
+ fd_ = -1;
+ ownsFd_ = false;
+}
+
+inline void File::swap(File& other) {
+ using std::swap;
+ swap(fd_, other.fd_);
+ swap(ownsFd_, other.ownsFd_);
+}
+
+inline File::File(File&& other) : fd_(other.fd_), ownsFd_(other.ownsFd_) {
+ other.release();
+}
+
+inline File& File::operator=(File&& other) {
+ File(std::move(other)).swap(*this);
+ return *this;
+}
+
+inline void swap(File& a, File& b) {
+ a.swap(b);
+}
+
+} // namespace folly
+
--- /dev/null
+/*
+ * Copyright 2012 Facebook, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "folly/experimental/File.h"
+
+#include <system_error>
+
+#include <glog/logging.h>
+
+namespace folly {
+
+File::File(const char* name, int flags, mode_t mode)
+ : fd_(::open(name, flags, mode)), ownsFd_(false) {
+ if (fd_ == -1) {
+ throw std::system_error(errno, std::system_category(), "open() failed");
+ }
+ ownsFd_ = true;
+}
+
+void File::close() {
+ if (!closeNoThrow()) {
+ throw std::system_error(errno, std::system_category(), "close() failed");
+ }
+}
+
+bool File::closeNoThrow() {
+ DCHECK(fd_ != -1);
+ int r = ownsFd_ ? ::close(fd_) : 0;
+ release();
+ return r == 0;
+}
+
+} // namespace folly
+
--- /dev/null
+/*
+ * Copyright 2012 Facebook, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef FOLLY_FILE_H_
+#define FOLLY_FILE_H_
+
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <fcntl.h>
+
+namespace folly {
+
+/**
+ * A File represents an open file.
+ */
+class File {
+ public:
+ /**
+ * Create a File object from an existing file descriptor.
+ * Takes ownership of the file descriptor if ownsFd is true.
+ */
+ /* implicit */ File(int fd, bool ownsFd=false);
+
+ /**
+ * Open and create a file object. Throws on error.
+ */
+ /* implicit */ File(const char* name, int flags=O_RDONLY, mode_t mode=0644);
+
+ ~File();
+
+ /**
+ * Return the file descriptor, or -1 if the file was closed.
+ */
+ int fd() const { return fd_; }
+
+ /**
+ * If we own the file descriptor, close the file and throw on error.
+ * Otherwise, do nothing.
+ */
+ void close();
+
+ /**
+ * Closes the file (if owned). Returns true on success, false (and sets
+ * errno) on error.
+ */
+ bool closeNoThrow();
+
+ /**
+ * Releases the file descriptor; no longer owned by this File.
+ */
+ void release();
+
+ /**
+ * Swap this File with another.
+ */
+ void swap(File& other);
+
+ // movable
+ File(File&&);
+ File& operator=(File&&);
+ private:
+ // not copyable
+ File(const File&) = delete;
+ File& operator=(const File&) = delete;
+
+ int fd_;
+ bool ownsFd_;
+};
+
+void swap(File& a, File& b);
+
+} // namespace folly
+
+#include "folly/experimental/File-inl.h"
+
+#endif /* FOLLY_FILE_H_ */
+
--- /dev/null
+/*
+ * Copyright 2012 Facebook, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef FOLLY_FILEGEN_H_
+#error This file may only be included from folly/experimental/FileGen.h
+#endif
+
+#include <system_error>
+
+#include "folly/experimental/StringGen.h"
+
+namespace folly {
+namespace gen {
+namespace detail {
+
+class FileReader : public GenImpl<ByteRange, FileReader> {
+ public:
+ FileReader(File file, std::unique_ptr<IOBuf> buffer)
+ : file_(std::move(file)),
+ buffer_(std::move(buffer)) {
+ buffer_->clear();
+ }
+
+ template <class Body>
+ bool apply(Body&& body) const {
+ for (;;) {
+ ssize_t n = ::read(file_.fd(), buffer_->writableTail(),
+ buffer_->capacity());
+ if (n == -1) {
+ throw std::system_error(errno, std::system_category(), "read failed");
+ }
+ if (n == 0) {
+ return true;
+ }
+ if (!body(ByteRange(buffer_->tail(), n))) {
+ return false;
+ }
+ }
+ }
+ private:
+ File file_;
+ std::unique_ptr<IOBuf> buffer_;
+};
+
+class FileWriter : public Operator<FileWriter> {
+ public:
+ FileWriter(File file, std::unique_ptr<IOBuf> buffer)
+ : file_(std::move(file)),
+ buffer_(std::move(buffer)) {
+ if (buffer_) {
+ buffer_->clear();
+ }
+ }
+
+ template <class Source, class Value>
+ void compose(const GenImpl<Value, Source>& source) const {
+ auto fn = [&](ByteRange v) {
+ if (!this->buffer_ || v.size() >= this->buffer_->capacity()) {
+ this->flushBuffer();
+ this->write(v);
+ } else {
+ if (v.size() > this->buffer_->tailroom()) {
+ this->flushBuffer();
+ }
+ memcpy(this->buffer_->writableTail(), v.data(), v.size());
+ this->buffer_->append(v.size());
+ }
+ };
+
+ // Iterate
+ source.foreach(std::move(fn));
+
+ flushBuffer();
+ file_.close();
+ }
+
+ private:
+ void write(ByteRange v) const {
+ ssize_t n;
+ while (!v.empty()) {
+ n = ::write(file_.fd(), v.data(), v.size());
+ if (n == -1) {
+ if (errno == EINTR) {
+ continue;
+ }
+ throw std::system_error(errno, std::system_category(),
+ "write() failed");
+ }
+ v.advance(n);
+ }
+ }
+
+ void flushBuffer() const {
+ if (buffer_ && buffer_->length() != 0) {
+ write(ByteRange(buffer_->data(), buffer_->length()));
+ buffer_->clear();
+ }
+ }
+
+ mutable File file_;
+ std::unique_ptr<IOBuf> buffer_;
+};
+
+} // namespace detail
+
+auto byLine(File file, char delim='\n') ->
+decltype(fromFile(std::move(file)) | eachAs<StringPiece>() | resplit(delim)) {
+ return fromFile(std::move(file)) | eachAs<StringPiece>() | resplit(delim);
+}
+
+} // namespace gen
+} // namespace folly
+
--- /dev/null
+/*
+ * Copyright 2012 Facebook, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef FOLLY_FILEGEN_H_
+#define FOLLY_FILEGEN_H_
+
+#include "folly/experimental/File.h"
+#include "folly/experimental/Gen.h"
+#include "folly/experimental/io/IOBuf.h"
+
+namespace folly {
+namespace gen {
+
+namespace detail {
+class FileReader;
+class FileWriter;
+} // namespace detail
+
+/**
+ * Generator that reads from a file with a buffer of the given size.
+ * Reads must be buffered (the generator interface expects the generator
+ * to hold each value).
+ */
+template <class S = detail::FileReader>
+S fromFile(File file, size_t bufferSize=4096) {
+ return S(std::move(file), IOBuf::create(bufferSize));
+}
+
+/**
+ * Generator that reads from a file using a given buffer.
+ */
+template <class S = detail::FileReader>
+S fromFile(File file, std::unique_ptr<IOBuf> buffer) {
+ return S(std::move(file), std::move(buffer));
+}
+
+/**
+ * Sink that writes to a file with a buffer of the given size.
+ * If bufferSize is 0, writes will be unbuffered.
+ */
+template <class S = detail::FileWriter>
+S toFile(File file, size_t bufferSize=4096) {
+ return S(std::move(file), bufferSize ? nullptr : IOBuf::create(bufferSize));
+}
+
+/**
+ * Sink that writes to a file using a given buffer.
+ * If the buffer is nullptr, writes will be unbuffered.
+ */
+template <class S = detail::FileWriter>
+S toFile(File file, std::unique_ptr<IOBuf> buffer) {
+ return S(std::move(file), std::move(buffer));
+}
+
+} // namespace gen
+} // namespace folly
+
+#include "folly/experimental/FileGen-inl.h"
+
+#endif /* FOLLY_FILEGEN_H_ */
+
class Gen,
class Op>
auto operator|(const GenImpl<Value, Gen>& gen, const Operator<Op>& op) ->
-decltype(op.self().compose(gen)) {
+decltype(op.self().compose(gen.self())) {
return op.self().compose(gen.self());
}
class Value>
class ReferencedSource :
public GenImpl<Value, ReferencedSource<Container, Value>> {
- Container* const container_;
+ Container* container_;
public:
explicit ReferencedSource(Container* container)
: container_(container) {}
static_assert(
!std::is_reference<Container>::value,
"Can't copy into a reference");
- const std::shared_ptr<const Container> copy_;
+ std::shared_ptr<const Container> copy_;
public:
typedef Container ContainerType;
template<class Value, class First, class Second>
class Chain : public GenImpl<Value,
Chain<Value, First, Second>> {
- const First first_;
- const Second second_;
+ First first_;
+ Second second_;
public:
explicit Chain(First first, Second second)
: first_(std::move(first))
**/
template<class Value, class Source>
class Yield : public GenImpl<Value, Yield<Value, Source>> {
- const Source source_;
+ Source source_;
public:
explicit Yield(Source source)
: source_(std::move(source)) {
*/
template<class Predicate>
class Map : public Operator<Map<Predicate>> {
- const Predicate predicate_;
+ Predicate predicate_;
public:
explicit Map(const Predicate& predicate = Predicate())
: predicate_(predicate)
>::type>
class Generator :
public GenImpl<Result, Generator<Value, Source, Result>> {
- const Source source_;
- const Predicate pred_;
+ Source source_;
+ Predicate pred_;
public:
explicit Generator(Source source, const Predicate& pred)
: source_(std::move(source)), pred_(pred) {}
*/
template<class Predicate>
class Filter : public Operator<Filter<Predicate>> {
- const Predicate predicate_;
+ Predicate predicate_;
public:
explicit Filter(const Predicate& predicate)
: predicate_(predicate)
template<class Value,
class Source>
class Generator : public GenImpl<Value, Generator<Value, Source>> {
- const Source source_;
- const Predicate pred_;
+ Source source_;
+ Predicate pred_;
public:
explicit Generator(Source source, const Predicate& pred)
: source_(std::move(source)), pred_(pred) {}
*/
template<class Predicate>
class Until : public Operator<Until<Predicate>> {
- const Predicate predicate_;
+ Predicate predicate_;
public:
explicit Until(const Predicate& predicate)
: predicate_(predicate)
class Result = typename std::result_of<Predicate(Value)>::type>
class Generator :
public GenImpl<Result, Generator<Value, Source, Result>> {
- const Source source_;
- const Predicate pred_;
+ Source source_;
+ Predicate pred_;
public:
explicit Generator(Source source, const Predicate& pred)
: source_(std::move(source)), pred_(pred) {}
* | take(10);
*/
class Take : public Operator<Take> {
- const size_t count_;
+ size_t count_;
public:
explicit Take(size_t count)
: count_(count) {}
class Source>
class Generator :
public GenImpl<Value, Generator<Value, Source>> {
- const Source source_;
- const size_t count_;
+ Source source_;
+ size_t count_;
public:
explicit Generator(Source source, size_t count)
: source_(std::move(source)) , count_(count) {}
* | take(10);
*/
class Skip : public Operator<Skip> {
- const size_t count_;
+ size_t count_;
public:
explicit Skip(size_t count)
: count_(count) {}
class Source>
class Generator :
public GenImpl<Value, Generator<Value, Source>> {
- const Source source_;
- const size_t count_;
+ Source source_;
+ size_t count_;
public:
explicit Generator(Source source, size_t count)
: source_(std::move(source)) , count_(count) {}
*/
template<class Selector, class Comparer>
class Order : public Operator<Order<Selector, Comparer>> {
- const Selector selector_;
- const Comparer comparer_;
+ Selector selector_;
+ Comparer comparer_;
public:
Order(const Selector& selector = Selector(),
const Comparer& comparer = Comparer())
class Generator :
public GenImpl<StorageType&&,
Generator<Value, Source, StorageType, Result>> {
- const Source source_;
- const Selector selector_;
- const Comparer comparer_;
+ Source source_;
+ Selector selector_;
+ Comparer comparer_;
typedef std::vector<StorageType> VectorType;
template<class First,
class Second>
class Composed : public Operator<Composed<First, Second>> {
- const First first_;
- const Second second_;
+ First first_;
+ Second second_;
public:
Composed() {}
Composed(First first, Second second)
template<class Seed,
class Fold>
class FoldLeft : public Operator<FoldLeft<Seed, Fold>> {
- const Seed seed_;
- const Fold fold_;
+ Seed seed_;
+ Fold fold_;
public:
FoldLeft(const Seed& seed, const Fold& fold)
: seed_(seed)
*/
template<class Reducer>
class Reduce : public Operator<Reduce<Reducer>> {
- const Reducer reducer_;
+ Reducer reducer_;
public:
Reduce(const Reducer& reducer)
: reducer_(reducer)
*/
template<class Collection>
class Append : public Operator<Append<Collection>> {
- Collection* const collection_;
+ Collection* collection_;
public:
explicit Append(Collection* collection)
: collection_(collection)
class InnerValue = typename std::decay<Inner>::type::ValueType>
class Generator :
public GenImpl<InnerValue, Generator<Inner, Source, InnerValue>> {
- const Source source_;
+ Source source_;
public:
explicit Generator(Source source)
: source_(std::move(source)) {}
class InnerValue = typename ValueTypeOfRange<Range>::RefType>
class Generator
: public GenImpl<InnerValue, Generator<Source, Range, InnerValue>> {
- const Source source_;
+ Source source_;
public:
explicit Generator(Source source)
: source_(std::move(source)) {}
template<class Wrapped>
class WrapperImpl : public WrapperBase {
- const Wrapped wrapped_;
+ Wrapped wrapped_;
public:
explicit WrapperImpl(Wrapped wrapped)
: wrapped_(std::move(wrapped)) {
#include "folly/Range.h"
#include "folly/Optional.h"
+#include "folly/Conv.h"
/**
* Generator-based Sequence Comprehensions in C++, akin to C#'s LINQ
}
};
+template <class Dest>
+class Cast {
+ public:
+ template <class Value>
+ Dest operator()(Value&& value) const {
+ return Dest(std::forward<Value>(value));
+ }
+};
+
+template <class Dest>
+class To {
+ public:
+ template <class Value>
+ Dest operator()(Value&& value) const {
+ return ::folly::to<Dest>(std::forward<Value>(value));
+ }
+};
+
namespace detail {
template<class Self>
return Get();
}
+// construct Dest from each value
+template <class Dest,
+ class Cast = detail::Map<Cast<Dest>>>
+Cast eachAs() {
+ return Cast();
+}
+
+// call folly::to on each value
+template <class Dest,
+ class To = detail::Map<To<Dest>>>
+To eachTo() {
+ return To();
+}
+
/*
* Sink Factories
*/
--- /dev/null
+/*
+ * Copyright 2012 Facebook, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef FOLLY_STRINGGEN_H_
+#error This file may only be included from folly/experimental/StringGen.h
+#endif
+
+#include "folly/experimental/io/IOBuf.h"
+
+namespace folly {
+namespace gen {
+namespace detail {
+
+bool splitPrefix(StringPiece& in, StringPiece& prefix, char delimiter) {
+ auto p = static_cast<const char*>(memchr(in.data(), delimiter, in.size()));
+ if (p) {
+ prefix.assign(in.data(), p);
+ in.assign(p + 1, in.end());
+ return true;
+ }
+ prefix.clear();
+ return false;
+}
+
+inline const char* ch(const unsigned char* p) {
+ return reinterpret_cast<const char*>(p);
+}
+
+class StringResplitter : public Operator<StringResplitter> {
+ char delimiter_;
+ public:
+ explicit StringResplitter(char delimiter) : delimiter_(delimiter) { }
+
+ template <class Source>
+ class Generator : public GenImpl<StringPiece, Generator<Source>> {
+ Source source_;
+ char delimiter_;
+ static constexpr size_t kDefaultLineSize = 256;
+ public:
+ Generator(Source source, char delimiter)
+ : source_(std::move(source)), delimiter_(delimiter) { }
+
+ template <class Body>
+ bool apply(Body&& body) const {
+ std::unique_ptr<IOBuf> buffer;
+
+ auto fn = [&](StringPiece in) -> bool {
+ StringPiece prefix;
+ bool found = splitPrefix(in, prefix, this->delimiter_);
+ if (found && buffer && buffer->length() != 0) {
+ // Append to end of buffer, return line
+ if (!prefix.empty()) {
+ buffer->reserve(0, prefix.size());
+ memcpy(buffer->writableTail(), prefix.data(), prefix.size());
+ buffer->append(prefix.size());
+ }
+ if (!body(StringPiece(ch(buffer->data()), buffer->length()))) {
+ return false;
+ }
+ buffer->clear();
+ found = splitPrefix(in, prefix, this->delimiter_);
+ }
+ // Buffer is empty, return lines directly from input (no buffer)
+ while (found) {
+ if (!body(prefix)) {
+ return false;
+ }
+ found = splitPrefix(in, prefix, this->delimiter_);
+ }
+ if (!in.empty()) {
+ // Incomplete line left, append to buffer
+ if (!buffer) {
+ // Arbitrarily assume that we have half a line and get enough
+ // room for twice that.
+ buffer = IOBuf::create(std::max(kDefaultLineSize, 2 * in.size()));
+ }
+ buffer->reserve(0, in.size());
+ memcpy(buffer->writableTail(), in.data(), in.size());
+ buffer->append(in.size());
+ }
+ return true;
+ };
+
+ // Iterate
+ if (!source_.apply(std::move(fn))) {
+ return false;
+ }
+
+ // Incomplete last line
+ if (buffer && buffer->length() != 0) {
+ if (!body(StringPiece(ch(buffer->data()), buffer->length()))) {
+ return false;
+ }
+ }
+ return true;
+ }
+ };
+
+ template<class Source,
+ class Value,
+ class Gen = Generator<Source>>
+ Gen compose(GenImpl<Value, Source>&& source) const {
+ return Gen(std::move(source.self()), delimiter_);
+ }
+
+ template<class Source,
+ class Value,
+ class Gen = Generator<Source>>
+ Gen compose(const GenImpl<Value, Source>& source) const {
+ return Gen(source.self(), delimiter_);
+ }
+};
+
+} // namespace detail
+} // namespace gen
+} // namespace folly
+
--- /dev/null
+/*
+ * Copyright 2012 Facebook, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef FOLLY_STRINGGEN_H_
+#define FOLLY_STRINGGEN_H_
+
+#include "folly/Range.h"
+
+namespace folly {
+namespace gen {
+
+namespace detail {
+class StringResplitter;
+} // namespace detail
+
+/**
+ * Split the output from a generator into StringPiece "lines" delimited by
+ * the given delimiter. Delimters are NOT included in the output.
+ *
+ * resplit() behaves as if the input strings were concatenated into one long
+ * string and then split.
+ */
+// make this a template so we don't require StringResplitter to be complete
+// until use
+template <class S=detail::StringResplitter>
+S resplit(char delimiter) {
+ return S(delimiter);
+}
+
+} // namespace gen
+} // namespace folly
+
+#include "folly/experimental/StringGen-inl.h"
+
+#endif /* FOLLY_STRINGGEN_H_ */
+
*/
#include "folly/experimental/Gen.h"
+#include "folly/experimental/StringGen.h"
+#include "folly/experimental/FileGen.h"
-#include <glog/logging.h>
#include <atomic>
+#include <thread>
+
+#include <glog/logging.h>
#include "folly/Benchmark.h"
folly::doNotOptimizeAway(s);
}
+BENCHMARK_DRAW_LINE()
+
+namespace {
+
+const char* const kLine = "The quick brown fox jumped over the lazy dog.\n";
+const size_t kLineCount = 10000;
+std::string bigLines;
+const size_t kSmallLineSize = 17;
+std::vector<std::string> smallLines;
+
+void initStringResplitterBenchmark() {
+ bigLines.reserve(kLineCount * strlen(kLine));
+ for (size_t i = 0; i < kLineCount; ++i) {
+ bigLines += kLine;
+ }
+ size_t remaining = bigLines.size();
+ size_t pos = 0;
+ while (remaining) {
+ size_t n = std::min(kSmallLineSize, remaining);
+ smallLines.push_back(bigLines.substr(pos, n));
+ pos += n;
+ remaining -= n;
+ }
+}
+
+size_t len(folly::StringPiece s) { return s.size(); }
+
+} // namespace
+
+BENCHMARK(StringResplitter_Big, iters) {
+ size_t s = 0;
+ while (iters--) {
+ s += from({bigLines}) | resplit('\n') | map(&len) | sum;
+ }
+ folly::doNotOptimizeAway(s);
+}
+
+BENCHMARK_RELATIVE(StringResplitter_Small, iters) {
+ size_t s = 0;
+ while (iters--) {
+ s += from(smallLines) | resplit('\n') | map(&len) | sum;
+ }
+ folly::doNotOptimizeAway(s);
+}
+
+BENCHMARK_DRAW_LINE()
+
+BENCHMARK(ByLine_Pipes, iters) {
+ std::thread thread;
+ int rfd;
+ int wfd;
+ BENCHMARK_SUSPEND {
+ int p[2];
+ CHECK_ERR(::pipe(p));
+ rfd = p[0];
+ wfd = p[1];
+ thread = std::thread([wfd, iters] {
+ char x = 'x';
+ PCHECK(::write(wfd, &x, 1) == 1); // signal startup
+ FILE* f = fdopen(wfd, "w");
+ PCHECK(f);
+ for (int i = 1; i <= iters; ++i) {
+ fprintf(f, "%d\n", i);
+ }
+ fclose(f);
+ });
+ char buf;
+ PCHECK(::read(rfd, &buf, 1) == 1); // wait for startup
+ }
+
+ auto s = byLine(rfd) | eachTo<int64_t>() | sum;
+ folly::doNotOptimizeAway(s);
+
+ BENCHMARK_SUSPEND {
+ ::close(rfd);
+ CHECK_EQ(s, int64_t(iters) * (iters + 1) / 2);
+ thread.join();
+ }
+}
+
// Results from a dual core Xeon L5520 @ 2.27GHz:
//
// ============================================================================
int main(int argc, char *argv[]) {
google::ParseCommandLineFlags(&argc, &argv, true);
+ initStringResplitterBenchmark();
runBenchmarks();
return 0;
}
#include <set>
#include <vector>
#include "folly/experimental/Gen.h"
+#include "folly/experimental/StringGen.h"
+#include "folly/experimental/FileGen.h"
+#include "folly/experimental/TestUtil.h"
#include "folly/FBVector.h"
+#include "folly/Format.h"
#include "folly/dynamic.h"
using namespace folly::gen;
using std::set;
using std::unique_ptr;
using std::vector;
+using std::string;
using std::tuple;
using std::make_tuple;
//using std::unordered_map;
EXPECT_EQ(6, gen | take(3) | sum);
}
+namespace {
+class TestIntSeq : public GenImpl<int, TestIntSeq> {
+ public:
+ TestIntSeq() { }
+
+ template <class Body>
+ bool apply(Body&& body) const {
+ for (int i = 1; i < 6; ++i) {
+ if (!body(i)) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ TestIntSeq(TestIntSeq&&) = default;
+ TestIntSeq& operator=(TestIntSeq&&) = default;
+ TestIntSeq(const TestIntSeq&) = delete;
+ TestIntSeq& operator=(const TestIntSeq&) = delete;
+};
+} // namespace
+
+TEST(Gen, NoGeneratorCopies) {
+ EXPECT_EQ(15, TestIntSeq() | sum);
+ auto x = TestIntSeq() | take(3);
+ EXPECT_EQ(6, std::move(x) | sum);
+}
+
TEST(Gen, FromArray) {
int source[] = {2, 3, 5, 7};
auto gen = from(source);
EXPECT_EQ(dynamic(5), from(array3) | rconcat | rconcat | sum);
}
+TEST(StringGen, EmptySplit) {
+ auto collect = eachTo<std::string>() | as<vector>();
+ {
+ auto pieces = from({""}) | resplit(',') | collect;
+ EXPECT_EQ(0, pieces.size());
+ }
+
+ // The last delimiter is eaten, just like std::getline
+ {
+ auto pieces = from({","}) | resplit(',') | collect;
+ EXPECT_EQ(1, pieces.size());
+ EXPECT_EQ("", pieces[0]);
+ }
+
+ {
+ auto pieces = from({",,"}) | resplit(',') | collect;
+ EXPECT_EQ(2, pieces.size());
+ EXPECT_EQ("", pieces[0]);
+ EXPECT_EQ("", pieces[1]);
+ }
+}
+
+TEST(StringGen, Split) {
+ auto collect = eachTo<std::string>() | as<vector>();
+ {
+ auto pieces = from({"hello,, world, goodbye, meow"}) |
+ resplit(',') | collect;
+ EXPECT_EQ(5, pieces.size());
+ EXPECT_EQ("hello", pieces[0]);
+ EXPECT_EQ("", pieces[1]);
+ EXPECT_EQ(" world", pieces[2]);
+ EXPECT_EQ(" goodbye", pieces[3]);
+ EXPECT_EQ(" meow", pieces[4]);
+ }
+ {
+ auto pieces = from({"hel", "lo,", ", world", ", goodbye, m", "eow"}) |
+ resplit(',') | collect;
+ EXPECT_EQ(5, pieces.size());
+ EXPECT_EQ("hello", pieces[0]);
+ EXPECT_EQ("", pieces[1]);
+ EXPECT_EQ(" world", pieces[2]);
+ EXPECT_EQ(" goodbye", pieces[3]);
+ EXPECT_EQ(" meow", pieces[4]);
+ }
+}
+
+TEST(FileGen, ByLine) {
+ auto collect = eachTo<std::string>() | as<vector>();
+ test::TemporaryFile file("ByLine");
+ static const std::string lines(
+ "Hello world\n"
+ "This is the second line\n"
+ "\n"
+ "\n"
+ "a few empty lines above\n"
+ "incomplete last line");
+ EXPECT_EQ(lines.size(), write(file.fd(), lines.data(), lines.size()));
+
+ auto expected = from({lines}) | resplit('\n') | collect;
+ auto found = byLine(file.path().c_str()) | collect;
+
+ EXPECT_TRUE(expected == found);
+}
+
+class FileGenBufferedTest : public ::testing::TestWithParam<int> { };
+
+TEST_P(FileGenBufferedTest, FileWriter) {
+ size_t bufferSize = GetParam();
+ test::TemporaryFile file("FileWriter");
+
+ static const std::string lines(
+ "Hello world\n"
+ "This is the second line\n"
+ "\n"
+ "\n"
+ "a few empty lines above\n");
+
+ auto src = from({lines, lines, lines, lines, lines, lines, lines, lines});
+ auto collect = eachTo<std::string>() | as<vector>();
+ auto expected = src | resplit('\n') | collect;
+
+ src | eachAs<StringPiece>() | toFile(file.fd(), bufferSize);
+ auto found = byLine(file.path().c_str()) | collect;
+
+ EXPECT_TRUE(expected == found);
+}
+
+INSTANTIATE_TEST_CASE_P(
+ DifferentBufferSizes,
+ FileGenBufferedTest,
+ ::testing::Values(0, 1, 2, 4, 8, 64, 4096));
+
int main(int argc, char *argv[]) {
testing::InitGoogleTest(&argc, argv);
google::ParseCommandLineFlags(&argc, &argv, true);
projects / folly.git / commitdiff