2 * Copyright 2017 Facebook, Inc.
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
17 #include <glog/logging.h>
23 #include <folly/gen/String.h>
24 #include <folly/portability/GTest.h>
26 using namespace folly::gen;
27 using namespace folly;
28 using std::make_tuple;
33 using std::unique_ptr;
36 TEST(StringGen, EmptySplit) {
37 auto collect = eachTo<std::string>() | as<vector>();
39 auto pieces = split("", ',') | collect;
40 EXPECT_EQ(0, pieces.size());
43 // The last delimiter is eaten, just like std::getline
45 auto pieces = split(",", ',') | collect;
46 EXPECT_EQ(1, pieces.size());
47 EXPECT_EQ("", pieces[0]);
51 auto pieces = split(",,", ',') | collect;
52 EXPECT_EQ(2, pieces.size());
53 EXPECT_EQ("", pieces[0]);
54 EXPECT_EQ("", pieces[1]);
58 auto pieces = split(",,", ',') | take(1) | collect;
59 EXPECT_EQ(1, pieces.size());
60 EXPECT_EQ("", pieces[0]);
64 TEST(StringGen, Split) {
65 auto collect = eachTo<std::string>() | as<vector>();
67 auto pieces = split("hello,, world, goodbye, meow", ',') | collect;
68 EXPECT_EQ(5, pieces.size());
69 EXPECT_EQ("hello", pieces[0]);
70 EXPECT_EQ("", pieces[1]);
71 EXPECT_EQ(" world", pieces[2]);
72 EXPECT_EQ(" goodbye", pieces[3]);
73 EXPECT_EQ(" meow", pieces[4]);
77 auto pieces = split("hello,, world, goodbye, meow", ',')
79 EXPECT_EQ(3, pieces.size());
80 EXPECT_EQ("hello", pieces[0]);
81 EXPECT_EQ("", pieces[1]);
82 EXPECT_EQ(" world", pieces[2]);
86 auto pieces = split("hello,, world, goodbye, meow", ",")
88 EXPECT_EQ(5, pieces.size());
89 EXPECT_EQ("hello", pieces[0]);
90 EXPECT_EQ("", pieces[1]);
91 EXPECT_EQ(" world", pieces[2]);
95 auto pieces = split("hello,, world, goodbye, meow", ", ")
97 EXPECT_EQ(4, pieces.size());
98 EXPECT_EQ("hello,", pieces[0]);
99 EXPECT_EQ("world", pieces[1]);
100 EXPECT_EQ("goodbye", pieces[2]);
101 EXPECT_EQ("meow", pieces[3]);
105 TEST(StringGen, SplitByNewLine) {
106 auto collect = eachTo<std::string>() | as<vector>();
108 auto pieces = lines("hello\n\n world\r\n goodbye\r me\n\row") | collect;
109 EXPECT_EQ(7, pieces.size());
110 EXPECT_EQ("hello", pieces[0]);
111 EXPECT_EQ("", pieces[1]);
112 EXPECT_EQ(" world", pieces[2]);
113 EXPECT_EQ(" goodbye", pieces[3]);
114 EXPECT_EQ(" me", pieces[4]);
115 EXPECT_EQ("", pieces[5]);
116 EXPECT_EQ("ow", pieces[6]);
120 TEST(StringGen, EmptyResplit) {
121 auto collect = eachTo<std::string>() | as<vector>();
123 auto pieces = from({""}) | resplit(',') | collect;
124 EXPECT_EQ(0, pieces.size());
127 // The last delimiter is eaten, just like std::getline
129 auto pieces = from({","}) | resplit(',') | collect;
130 EXPECT_EQ(1, pieces.size());
131 EXPECT_EQ("", pieces[0]);
135 auto pieces = from({",,"}) | resplit(',') | collect;
136 EXPECT_EQ(2, pieces.size());
137 EXPECT_EQ("", pieces[0]);
138 EXPECT_EQ("", pieces[1]);
142 TEST(StringGen, EachToTuple) {
144 auto lines = "2:1.414:yo 3:1.732:hi";
147 | eachToTuple<int, double, std::string>(':')
149 vector<tuple<int, double, std::string>> expected {
150 make_tuple(2, 1.414, "yo"),
151 make_tuple(3, 1.732, "hi"),
153 EXPECT_EQ(expected, actual);
159 | eachToTuple<int>(',')
161 vector<tuple<int>> expected {
165 EXPECT_EQ(expected, actual);
168 // StringPiece target
169 auto lines = "1:cat 2:dog";
172 | eachToTuple<int, StringPiece>(':')
174 vector<tuple<int, StringPiece>> expected {
175 make_tuple(1, "cat"),
176 make_tuple(2, "dog"),
178 EXPECT_EQ(expected, actual);
182 auto lines = "2:tjackson:4 3::5";
185 | eachToTuple<int, fbstring, int>(':')
187 vector<tuple<int, fbstring, int>> expected {
188 make_tuple(2, "tjackson", 4),
189 make_tuple(3, "", 5),
191 EXPECT_EQ(expected, actual);
195 auto lines = "1:2 3:4:5";
196 EXPECT_THROW((split(lines, ' ')
197 | eachToTuple<int, int>(':')
203 auto lines = "1:2:3 4:5";
204 EXPECT_THROW((split(lines, ' ')
205 | eachToTuple<int, int, int>(':')
211 TEST(StringGen, EachToPair) {
214 auto lines = "2:1.414 3:1.732";
217 | eachToPair<int, double>(':')
218 | as<std::map<int, double>>();
219 std::map<int, double> expected {
223 EXPECT_EQ(expected, actual);
227 auto lines = "ab=>cd ef=>gh";
230 | eachToPair<string, string>("=>")
231 | as<std::map<string, string>>();
232 std::map<string, string> expected {
236 EXPECT_EQ(expected, actual);
240 TEST(StringGen, Resplit) {
241 auto collect = eachTo<std::string>() | as<vector>();
243 auto pieces = from({"hello,, world, goodbye, meow"}) |
244 resplit(',') | collect;
245 EXPECT_EQ(5, pieces.size());
246 EXPECT_EQ("hello", pieces[0]);
247 EXPECT_EQ("", pieces[1]);
248 EXPECT_EQ(" world", pieces[2]);
249 EXPECT_EQ(" goodbye", pieces[3]);
250 EXPECT_EQ(" meow", pieces[4]);
253 auto pieces = from({"hel", "lo,", ", world", ", goodbye, m", "eow"}) |
254 resplit(',') | collect;
255 EXPECT_EQ(5, pieces.size());
256 EXPECT_EQ("hello", pieces[0]);
257 EXPECT_EQ("", pieces[1]);
258 EXPECT_EQ(" world", pieces[2]);
259 EXPECT_EQ(" goodbye", pieces[3]);
260 EXPECT_EQ(" meow", pieces[4]);
264 void checkResplitMaxLength(vector<string> ins,
267 vector<string> outs) {
268 vector<std::string> pieces;
269 auto splitter = streamSplitter(delim, [&pieces](StringPiece s) {
270 pieces.push_back(string(s.begin(), s.end()));
273 for (const auto& in : ins) {
278 EXPECT_EQ(outs.size(), pieces.size());
279 for (size_t i = 0; i < outs.size(); ++i) {
280 EXPECT_EQ(outs[i], pieces[i]);
283 // Also check the concatenated input against the same output
284 if (ins.size() > 1) {
285 checkResplitMaxLength({folly::join("", ins)}, delim, maxLength, outs);
289 TEST(StringGen, ResplitMaxLength) {
290 checkResplitMaxLength(
291 {"hel", "lo,", ", world", ", goodbye, m", "ew"}, ',', 5,
292 {"hello", ",", ",", " worl", "d,", " good", "bye,", " mew"}
294 // " meow" cannot be "end of stream", since it's maxLength long
295 checkResplitMaxLength(
296 {"hel", "lo,", ", world", ", goodbye, m", "eow"}, ',', 5,
297 {"hello", ",", ",", " worl", "d,", " good", "bye,", " meow", ""}
299 checkResplitMaxLength(
300 {"||", "", "", "", "|a|b", "cdefghijklmn", "|opqrst",
301 "uvwx|y|||", "z", "0123456789", "|", ""}, '|', 2,
302 {"|", "|", "|", "a|", "bc", "de", "fg", "hi", "jk", "lm", "n|", "op", "qr",
303 "st", "uv", "wx", "|", "y|", "|", "|", "z0", "12", "34", "56", "78", "9|",
309 void runUnsplitSuite(F fn) {
311 fn("hello,world,goodbye");
318 TEST(StringGen, Unsplit) {
320 auto basicFn = [](StringPiece s) {
321 EXPECT_EQ(split(s, ',') | unsplit(','), s);
324 auto existingBuffer = [](StringPiece s) {
325 folly::fbstring buffer("asdf");
326 split(s, ',') | unsplit(',', &buffer);
327 auto expected = folly::to<folly::fbstring>(
328 "asdf", s.empty() ? "" : ",", s);
329 EXPECT_EQ(expected, buffer);
332 auto emptyBuffer = [](StringPiece s) {
334 split(s, ',') | unsplit(',', &buffer);
335 EXPECT_EQ(s, buffer);
338 auto stringDelim = [](StringPiece s) {
339 EXPECT_EQ(s, split(s, ',') | unsplit(","));
341 split(s, ',') | unsplit(",", &buffer);
342 EXPECT_EQ(buffer, s);
345 runUnsplitSuite(basicFn);
346 runUnsplitSuite(existingBuffer);
347 runUnsplitSuite(emptyBuffer);
348 runUnsplitSuite(stringDelim);
349 EXPECT_EQ("1, 2, 3", seq(1, 3) | unsplit(", "));
352 TEST(StringGen, Batch) {
353 std::vector<std::string> chunks{
354 "on", "e\nt", "w", "o", "\nthr", "ee\nfo", "ur\n",
356 std::vector<std::string> lines{
357 "one", "two", "three", "four",
359 EXPECT_EQ(4, from(chunks) | resplit('\n') | count);
360 EXPECT_EQ(4, from(chunks) | resplit('\n') | batch(2) | rconcat | count);
361 EXPECT_EQ(4, from(chunks) | resplit('\n') | batch(3) | rconcat | count);
362 EXPECT_EQ(lines, from(chunks) | resplit('\n') | eachTo<std::string>() |
363 batch(3) | rconcat | as<vector>());