2 * Copyright 2014 Facebook, Inc.
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
17 #ifndef FOLLY_GEN_STRING_H
18 #define FOLLY_GEN_STRING_H
20 #include <folly/Range.h>
21 #include <folly/gen/Base.h>
22 #include <folly/io/IOBuf.h>
28 class StringResplitter;
30 template<class Delimiter>
31 class SplitStringSource;
33 template<class Delimiter, class Output>
36 template<class Delimiter, class OutputBuffer>
39 template<class TargetContainer,
47 * Split the output from a generator into StringPiece "lines" delimited by
48 * the given delimiter. Delimters are NOT included in the output.
50 * resplit() behaves as if the input strings were concatenated into one long
51 * string and then split.
53 * Equivalently, you can use StreamSplitter outside of a folly::gen setting.
55 // make this a template so we don't require StringResplitter to be complete
57 template <class S=detail::StringResplitter>
58 S resplit(char delimiter) {
62 template <class S = detail::SplitStringSource<char>>
63 S split(const StringPiece source, char delimiter) {
64 return S(source, delimiter);
67 template <class S = detail::SplitStringSource<StringPiece>>
68 S split(StringPiece source, StringPiece delimiter) {
69 return S(source, delimiter);
73 * EOL terms ("\r", "\n", or "\r\n").
75 class MixedNewlines {};
78 * Split by EOL ("\r", "\n", or "\r\n").
81 template <class S = detail::SplitStringSource<MixedNewlines>>
82 S lines(StringPiece source) {
83 return S(source, MixedNewlines{});
87 * Joins a sequence of tokens into a string, with the chosen delimiter.
90 * fbstring result = split("a,b,c", ",") | unsplit(",");
91 * assert(result == "a,b,c");
93 * std::string result = split("a,b,c", ",") | unsplit<std::string>(" ");
94 * assert(result == "a b c");
98 // NOTE: The template arguments are reversed to allow the user to cleanly
99 // specify the output type while still inferring the type of the delimiter.
100 template<class Output = folly::fbstring,
102 class Unsplit = detail::Unsplit<Delimiter, Output>>
103 Unsplit unsplit(const Delimiter& delimiter) {
104 return Unsplit(delimiter);
107 template<class Output = folly::fbstring,
108 class Unsplit = detail::Unsplit<fbstring, Output>>
109 Unsplit unsplit(const char* delimiter) {
110 return Unsplit(delimiter);
114 * Joins a sequence of tokens into a string, appending them to the output
115 * buffer. If the output buffer is empty, an initial delimiter will not be
116 * inserted at the start.
119 * std::string buffer;
120 * split("a,b,c", ",") | unsplit(",", &buffer);
121 * assert(buffer == "a,b,c");
123 * std::string anotherBuffer("initial");
124 * split("a,b,c", ",") | unsplit(",", &anotherbuffer);
125 * assert(anotherBuffer == "initial,a,b,c");
127 template<class Delimiter,
129 class UnsplitBuffer = detail::UnsplitBuffer<Delimiter, OutputBuffer>>
130 UnsplitBuffer unsplit(Delimiter delimiter, OutputBuffer* outputBuffer) {
131 return UnsplitBuffer(delimiter, outputBuffer);
134 template<class OutputBuffer,
135 class UnsplitBuffer = detail::UnsplitBuffer<fbstring, OutputBuffer>>
136 UnsplitBuffer unsplit(const char* delimiter, OutputBuffer* outputBuffer) {
137 return UnsplitBuffer(delimiter, outputBuffer);
141 template<class... Targets>
142 detail::Map<detail::SplitTo<std::tuple<Targets...>, char, Targets...>>
143 eachToTuple(char delim) {
145 detail::SplitTo<std::tuple<Targets...>, char, Targets...>>(
146 detail::SplitTo<std::tuple<Targets...>, char, Targets...>(delim));
149 template<class... Targets>
150 detail::Map<detail::SplitTo<std::tuple<Targets...>, fbstring, Targets...>>
151 eachToTuple(StringPiece delim) {
153 detail::SplitTo<std::tuple<Targets...>, fbstring, Targets...>>(
154 detail::SplitTo<std::tuple<Targets...>, fbstring, Targets...>(delim));
157 template<class First, class Second>
158 detail::Map<detail::SplitTo<std::pair<First, Second>, char, First, Second>>
159 eachToPair(char delim) {
161 detail::SplitTo<std::pair<First, Second>, char, First, Second>>(
162 detail::SplitTo<std::pair<First, Second>, char, First, Second>(delim));
165 template<class First, class Second>
166 detail::Map<detail::SplitTo<std::pair<First, Second>, fbstring, First, Second>>
167 eachToPair(StringPiece delim) {
169 detail::SplitTo<std::pair<First, Second>, fbstring, First, Second>>(
170 detail::SplitTo<std::pair<First, Second>, fbstring, First, Second>(
171 to<fbstring>(delim)));
175 * Outputs exactly the same bytes as the input stream, in different chunks.
176 * A chunk boundary occurs after each delimiter, or, if maxLength is
177 * non-zero, after maxLength bytes, whichever comes first. Your callback
178 * can return false to stop consuming the stream at any time.
180 * The splitter buffers the last incomplete chunk, so you must call flush()
181 * to consume the piece of the stream after the final delimiter. This piece
182 * may be empty. After a flush(), the splitter can be re-used for a new
185 * operator() and flush() return false iff your callback returns false. The
186 * internal buffer is not flushed, so reusing such a splitter will have
187 * indeterminate results. Same goes if your callback throws. Feel free to
188 * fix these corner cases if needed.
191 * - Create via streamSplitter() to take advantage of template deduction.
192 * - If your callback needs an end-of-stream signal, test for "no
193 * trailing delimiter **and** shorter than maxLength".
194 * - You can fine-tune the initial capacity of the internal IOBuf.
196 template <class Callback>
197 class StreamSplitter {
200 StreamSplitter(char delimiter,
202 uint64_t maxLength = 0,
203 uint64_t initialCapacity = 0)
204 : buffer_(IOBuf::CREATE, initialCapacity),
205 delimiter_(delimiter),
206 maxLength_(maxLength),
207 pieceCb_(std::move(pieceCb)) {}
210 * Consume any incomplete last line (may be empty). Do this before
211 * destroying the StreamSplitter, or you will fail to consume part of the
214 * After flush() you may proceed to consume the next stream via ().
216 * Returns false if the callback wants no more data, true otherwise.
217 * A return value of false means that this splitter must no longer be used.
222 * Consume another piece of the input stream.
224 * Returns false only if your callback refuses to consume more data by
225 * returning false (true otherwise). A return value of false means that
226 * this splitter must no longer be used.
228 bool operator()(StringPiece in);
231 // Holds the current "incomplete" chunk so that chunks can span calls to ()
234 uint64_t maxLength_; // The callback never gets more chars than this
238 template <class Callback> // Helper to enable template deduction
239 StreamSplitter<Callback> streamSplitter(char delimiter,
241 uint64_t capacity = 0) {
242 return StreamSplitter<Callback>(delimiter, std::move(pieceCb), capacity);
248 #include <folly/gen/String-inl.h>
250 #endif // FOLLY_GEN_STRING_H