2 * Copyright 2014-present Facebook, Inc.
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
21 #include <folly/functional/ApplyTuple.h>
22 #include <folly/gen/String.h>
23 #include <folly/portability/GTest.h>
25 using namespace folly::gen;
26 using namespace folly;
27 using std::make_tuple;
32 using std::unique_ptr;
35 using vec = vector<string>;
37 static auto collect = eachTo<std::string>() | as<vector>();
39 TEST(StringGen, EmptySplit) {
42 auto expected = vec{};
43 EXPECT_EQ(expected, split(input, ',') | collect);
46 // The last delimiter is eaten, just like std::getline
49 auto expected = vec{""};
50 EXPECT_EQ(expected, split(input, ',') | collect);
55 auto expected = vec{"", ""};
56 EXPECT_EQ(expected, split(input, ',') | collect);
61 auto expected = vec{""};
62 EXPECT_EQ(expected, split(input, ',') | take(1) | collect);
66 TEST(StringGen, Split) {
68 auto input = "hello,, world, goodbye, meow";
69 auto expected = vec{"hello", "", " world", " goodbye", " meow"};
70 EXPECT_EQ(expected, split(input, ',') | collect);
74 auto input = "hello,, world, goodbye, meow";
75 auto expected = vec{"hello", "", " world"};
76 EXPECT_EQ(expected, split(input, ',') | take(3) | collect);
80 auto input = "hello,, world, goodbye, meow";
81 auto expected = vec{"hello", "", " world", " goodbye", " meow"};
82 EXPECT_EQ(expected, split(input, ",") | take(5) | collect);
86 auto input = "hello,, world, goodbye, meow";
87 auto expected = vec{"hello,", "world", "goodbye", "meow"};
88 EXPECT_EQ(expected, split(input, ", ") | collect);
92 TEST(StringGen, SplitByNewLine) {
94 auto input = "hello\n\n world\r\n goodbye\r me\n\row";
95 auto expected = vec{"hello", "", " world", " goodbye", " me", "", "ow"};
96 EXPECT_EQ(expected, lines(input) | collect);
100 TEST(StringGen, EmptyResplit) {
102 auto input = vec{""};
103 auto expected = vec{};
104 EXPECT_EQ(expected, from(input) | resplit(',') | collect);
107 // The last delimiter is eaten, just like std::getline
109 auto input = vec{","};
110 auto expected = vec{""};
111 EXPECT_EQ(expected, from(input) | resplit(',') | collect);
115 auto input = vec{",,"};
116 auto expected = vec{"", ""};
117 EXPECT_EQ(expected, from(input) | resplit(',') | collect);
121 TEST(StringGen, Resplit) {
123 auto input = vec{"hello,, world, goodbye, meow"};
124 auto expected = vec{"hello", "", " world", " goodbye", " meow"};
125 EXPECT_EQ(expected, from(input) | resplit(',') | collect);
129 auto input = vec{"hel", "lo,", ", world", ", goodbye, m", "eow"};
130 auto expected = vec{"hello", "", " world", " goodbye", " meow"};
131 EXPECT_EQ(expected, from(input) | resplit(',') | collect);
135 TEST(StringGen, ResplitKeepDelimiter) {
137 auto input = vec{"hello,, world, goodbye, meow"};
138 auto expected = vec{"hello,", ",", " world,", " goodbye,", " meow"};
139 EXPECT_EQ(expected, from(input) | resplit(',', true) | collect);
143 auto input = vec{"hel", "lo,", ", world", ", goodbye, m", "eow"};
144 auto expected = vec{"hello,", ",", " world,", " goodbye,", " meow"};
145 EXPECT_EQ(expected, from(input) | resplit(',', true) | collect);
149 TEST(StringGen, EachToTuple) {
151 auto lines = "2:1.414:yo 3:1.732:hi";
154 | eachToTuple<int, double, std::string>(':')
156 vector<tuple<int, double, std::string>> expected {
157 make_tuple(2, 1.414, "yo"),
158 make_tuple(3, 1.732, "hi"),
160 EXPECT_EQ(expected, actual);
166 | eachToTuple<int>(',')
168 vector<tuple<int>> expected {
172 EXPECT_EQ(expected, actual);
175 // StringPiece target
176 auto lines = "1:cat 2:dog";
179 | eachToTuple<int, StringPiece>(':')
181 vector<tuple<int, StringPiece>> expected {
182 make_tuple(1, "cat"),
183 make_tuple(2, "dog"),
185 EXPECT_EQ(expected, actual);
189 auto lines = "2:tjackson:4 3::5";
192 | eachToTuple<int, fbstring, int>(':')
194 vector<tuple<int, fbstring, int>> expected {
195 make_tuple(2, "tjackson", 4),
196 make_tuple(3, "", 5),
198 EXPECT_EQ(expected, actual);
202 auto lines = "1:2 3:4:5";
203 EXPECT_THROW((split(lines, ' ')
204 | eachToTuple<int, int>(':')
210 auto lines = "1:2:3 4:5";
211 EXPECT_THROW((split(lines, ' ')
212 | eachToTuple<int, int, int>(':')
218 TEST(StringGen, EachToPair) {
221 auto lines = "2:1.414 3:1.732";
224 | eachToPair<int, double>(':')
225 | as<std::map<int, double>>();
226 std::map<int, double> expected {
230 EXPECT_EQ(expected, actual);
234 auto lines = "ab=>cd ef=>gh";
237 | eachToPair<string, string>("=>")
238 | as<std::map<string, string>>();
239 std::map<string, string> expected {
243 EXPECT_EQ(expected, actual);
247 void checkResplitMaxLength(vector<string> ins,
250 vector<string> outs) {
251 vector<std::string> pieces;
252 auto splitter = streamSplitter(delim, [&pieces](StringPiece s) {
253 pieces.push_back(string(s.begin(), s.end()));
256 for (const auto& in : ins) {
261 EXPECT_EQ(outs.size(), pieces.size());
262 for (size_t i = 0; i < outs.size(); ++i) {
263 EXPECT_EQ(outs[i], pieces[i]);
266 // Also check the concatenated input against the same output
267 if (ins.size() > 1) {
268 checkResplitMaxLength({folly::join("", ins)}, delim, maxLength, outs);
272 TEST(StringGen, ResplitMaxLength) {
273 checkResplitMaxLength(
274 {"hel", "lo,", ", world", ", goodbye, m", "ew"}, ',', 5,
275 {"hello", ",", ",", " worl", "d,", " good", "bye,", " mew"}
277 // " meow" cannot be "end of stream", since it's maxLength long
278 checkResplitMaxLength(
279 {"hel", "lo,", ", world", ", goodbye, m", "eow"}, ',', 5,
280 {"hello", ",", ",", " worl", "d,", " good", "bye,", " meow", ""}
282 checkResplitMaxLength(
283 {"||", "", "", "", "|a|b", "cdefghijklmn", "|opqrst",
284 "uvwx|y|||", "z", "0123456789", "|", ""}, '|', 2,
285 {"|", "|", "|", "a|", "bc", "de", "fg", "hi", "jk", "lm", "n|", "op", "qr",
286 "st", "uv", "wx", "|", "y|", "|", "|", "z0", "12", "34", "56", "78", "9|",
291 template <typename F>
292 void runUnsplitSuite(F fn) {
294 fn("hello,world,goodbye");
301 TEST(StringGen, Unsplit) {
303 auto basicFn = [](StringPiece s) {
304 EXPECT_EQ(split(s, ',') | unsplit(','), s);
307 auto existingBuffer = [](StringPiece s) {
308 folly::fbstring buffer("asdf");
309 split(s, ',') | unsplit(',', &buffer);
310 auto expected = folly::to<folly::fbstring>(
311 "asdf", s.empty() ? "" : ",", s);
312 EXPECT_EQ(expected, buffer);
315 auto emptyBuffer = [](StringPiece s) {
317 split(s, ',') | unsplit(',', &buffer);
318 EXPECT_EQ(s, buffer);
321 auto stringDelim = [](StringPiece s) {
322 EXPECT_EQ(s, split(s, ',') | unsplit(","));
324 split(s, ',') | unsplit(",", &buffer);
325 EXPECT_EQ(buffer, s);
328 runUnsplitSuite(basicFn);
329 runUnsplitSuite(existingBuffer);
330 runUnsplitSuite(emptyBuffer);
331 runUnsplitSuite(stringDelim);
332 EXPECT_EQ("1, 2, 3", seq(1, 3) | unsplit(", "));
335 TEST(StringGen, Batch) {
336 std::vector<std::string> chunks{
337 "on", "e\nt", "w", "o", "\nthr", "ee\nfo", "ur\n",
339 std::vector<std::string> lines{
340 "one", "two", "three", "four",
342 EXPECT_EQ(4, from(chunks) | resplit('\n') | count);
343 EXPECT_EQ(4, from(chunks) | resplit('\n') | batch(2) | rconcat | count);
344 EXPECT_EQ(4, from(chunks) | resplit('\n') | batch(3) | rconcat | count);
345 EXPECT_EQ(lines, from(chunks) | resplit('\n') | eachTo<std::string>() |
346 batch(3) | rconcat | as<vector>());
349 TEST(StringGen, UncurryTuple) {
350 folly::StringPiece file = "1\t2\t3\n1\t4\t9";
351 auto rows = split(file, '\n') | eachToTuple<int, int, int>('\t');
353 rows | map(uncurry([](int x, int y, int z) { return x * y * z; })) | sum;
354 EXPECT_EQ(42, productSum);
357 TEST(StringGen, UncurryPair) {
358 folly::StringPiece file = "2\t3\n4\t9";
359 auto rows = split(file, '\n') | eachToPair<int, int>('\t');
361 rows | map(uncurry([](int x, int y) { return x * y; })) | sum;
362 EXPECT_EQ(42, productSum);