2 * Copyright 2014-present Facebook, Inc.
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
18 #define FOLLY_GEN_STRING_H_
20 #include <folly/Range.h>
21 #include <folly/gen/Base.h>
22 #include <folly/io/IOBuf.h>
28 class StringResplitter;
30 template <class Delimiter>
31 class SplitStringSource;
33 template <class Delimiter, class Output>
36 template <class Delimiter, class OutputBuffer>
39 template <class TargetContainer, class Delimiter, class... Targets>
45 * Split the output from a generator into StringPiece "lines" delimited by
46 * the given delimiter. Delimters are NOT included in the output.
48 * resplit() behaves as if the input strings were concatenated into one long
49 * string and then split.
51 * Equivalently, you can use StreamSplitter outside of a folly::gen setting.
53 // make this a template so we don't require StringResplitter to be complete
55 template <class S = detail::StringResplitter>
56 S resplit(char delimiter, bool keepDelimiter = false) {
57 return S(delimiter, keepDelimiter);
60 template <class S = detail::SplitStringSource<char>>
61 S split(const StringPiece source, char delimiter) {
62 return S(source, delimiter);
65 template <class S = detail::SplitStringSource<StringPiece>>
66 S split(StringPiece source, StringPiece delimiter) {
67 return S(source, delimiter);
71 * EOL terms ("\r", "\n", or "\r\n").
73 class MixedNewlines {};
76 * Split by EOL ("\r", "\n", or "\r\n").
79 template <class S = detail::SplitStringSource<MixedNewlines>>
80 S lines(StringPiece source) {
81 return S(source, MixedNewlines{});
85 * Joins a sequence of tokens into a string, with the chosen delimiter.
88 * fbstring result = split("a,b,c", ",") | unsplit(",");
89 * assert(result == "a,b,c");
91 * std::string result = split("a,b,c", ",") | unsplit<std::string>(" ");
92 * assert(result == "a b c");
96 // NOTE: The template arguments are reversed to allow the user to cleanly
97 // specify the output type while still inferring the type of the delimiter.
99 class Output = folly::fbstring,
101 class Unsplit = detail::Unsplit<Delimiter, Output>>
102 Unsplit unsplit(const Delimiter& delimiter) {
103 return Unsplit(delimiter);
107 class Output = folly::fbstring,
108 class Unsplit = detail::Unsplit<fbstring, Output>>
109 Unsplit unsplit(const char* delimiter) {
110 return Unsplit(delimiter);
114 * Joins a sequence of tokens into a string, appending them to the output
115 * buffer. If the output buffer is empty, an initial delimiter will not be
116 * inserted at the start.
119 * std::string buffer;
120 * split("a,b,c", ",") | unsplit(",", &buffer);
121 * assert(buffer == "a,b,c");
123 * std::string anotherBuffer("initial");
124 * split("a,b,c", ",") | unsplit(",", &anotherbuffer);
125 * assert(anotherBuffer == "initial,a,b,c");
130 class UnsplitBuffer = detail::UnsplitBuffer<Delimiter, OutputBuffer>>
131 UnsplitBuffer unsplit(Delimiter delimiter, OutputBuffer* outputBuffer) {
132 return UnsplitBuffer(delimiter, outputBuffer);
137 class UnsplitBuffer = detail::UnsplitBuffer<fbstring, OutputBuffer>>
138 UnsplitBuffer unsplit(const char* delimiter, OutputBuffer* outputBuffer) {
139 return UnsplitBuffer(delimiter, outputBuffer);
142 template <class... Targets>
143 detail::Map<detail::SplitTo<std::tuple<Targets...>, char, Targets...>>
144 eachToTuple(char delim) {
146 detail::SplitTo<std::tuple<Targets...>, char, Targets...>>(
147 detail::SplitTo<std::tuple<Targets...>, char, Targets...>(delim));
150 template <class... Targets>
151 detail::Map<detail::SplitTo<std::tuple<Targets...>, fbstring, Targets...>>
152 eachToTuple(StringPiece delim) {
154 detail::SplitTo<std::tuple<Targets...>, fbstring, Targets...>>(
155 detail::SplitTo<std::tuple<Targets...>, fbstring, Targets...>(delim));
158 template <class First, class Second>
159 detail::Map<detail::SplitTo<std::pair<First, Second>, char, First, Second>>
160 eachToPair(char delim) {
162 detail::SplitTo<std::pair<First, Second>, char, First, Second>>(
163 detail::SplitTo<std::pair<First, Second>, char, First, Second>(delim));
166 template <class First, class Second>
167 detail::Map<detail::SplitTo<std::pair<First, Second>, fbstring, First, Second>>
168 eachToPair(StringPiece delim) {
170 detail::SplitTo<std::pair<First, Second>, fbstring, First, Second>>(
171 detail::SplitTo<std::pair<First, Second>, fbstring, First, Second>(
172 to<fbstring>(delim)));
176 * Outputs exactly the same bytes as the input stream, in different chunks.
177 * A chunk boundary occurs after each delimiter, or, if maxLength is
178 * non-zero, after maxLength bytes, whichever comes first. Your callback
179 * can return false to stop consuming the stream at any time.
181 * The splitter buffers the last incomplete chunk, so you must call flush()
182 * to consume the piece of the stream after the final delimiter. This piece
183 * may be empty. After a flush(), the splitter can be re-used for a new
186 * operator() and flush() return false iff your callback returns false. The
187 * internal buffer is not flushed, so reusing such a splitter will have
188 * indeterminate results. Same goes if your callback throws. Feel free to
189 * fix these corner cases if needed.
192 * - Create via streamSplitter() to take advantage of template deduction.
193 * - If your callback needs an end-of-stream signal, test for "no
194 * trailing delimiter **and** shorter than maxLength".
195 * - You can fine-tune the initial capacity of the internal IOBuf.
197 template <class Callback>
198 class StreamSplitter {
201 StreamSplitter(char delimiter,
203 uint64_t maxLength = 0,
204 uint64_t initialCapacity = 0)
205 : buffer_(IOBuf::CREATE, initialCapacity),
206 delimiter_(delimiter),
207 maxLength_(maxLength),
208 pieceCb_(std::move(pieceCb)) {}
211 * Consume any incomplete last line (may be empty). Do this before
212 * destroying the StreamSplitter, or you will fail to consume part of the
215 * After flush() you may proceed to consume the next stream via ().
217 * Returns false if the callback wants no more data, true otherwise.
218 * A return value of false means that this splitter must no longer be used.
223 * Consume another piece of the input stream.
225 * Returns false only if your callback refuses to consume more data by
226 * returning false (true otherwise). A return value of false means that
227 * this splitter must no longer be used.
229 bool operator()(StringPiece in);
232 // Holds the current "incomplete" chunk so that chunks can span calls to ()
235 uint64_t maxLength_; // The callback never gets more chars than this
239 template <class Callback> // Helper to enable template deduction
240 StreamSplitter<Callback> streamSplitter(char delimiter,
242 uint64_t capacity = 0) {
243 return StreamSplitter<Callback>(delimiter, std::move(pieceCb), capacity);
249 #include <folly/gen/String-inl.h>