2 * Copyright 2012 Facebook, Inc.
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
17 #include "folly/json.h"
19 #include <boost/next_prior.hpp>
20 #include <boost/algorithm/string.hpp>
22 #include "folly/Range.h"
23 #include "folly/Unicode.h"
24 #include "folly/Conv.h"
28 //////////////////////////////////////////////////////////////////////
33 char32_t decodeUtf8(const unsigned char*& p, const unsigned char* const e) {
34 /* The following encodings are valid, except for the 5 and 6 byte
38 * 1110xxxx 10xxxxxx 10xxxxxx
39 * 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
40 * 111110xx 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx
41 * 1111110x 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx
45 throw std::runtime_error("folly::decodeUtf8 empty/invalid string");
48 unsigned char fst = *p;
54 static const uint32_t bitMask[] = {
61 // upper control bits are masked out later
64 if ((fst & 0xC0) != 0xC0) {
65 throw std::runtime_error(
66 to<std::string>("folly::decodeUtf8 i=0 d=", d));
71 for (unsigned int i = 1; i != 3 && p + i < e; ++i) {
72 unsigned char tmp = p[i];
74 if ((tmp & 0xC0) != 0x80) {
75 throw std::runtime_error(
76 to<std::string>("folly::decodeUtf8 i=", i, " tmp=", (uint32_t)tmp));
79 d = (d << 6) | (tmp & 0x3F);
85 // overlong, could have been encoded with i bytes
86 if ((d & ~bitMask[i - 1]) == 0) {
87 throw std::runtime_error(
88 to<std::string>("folly::decodeUtf8 i=", i, " d=", d));
91 // check for surrogates only needed for 3 bytes
93 if ((d >= 0xD800 && d <= 0xDFFF) || d > 0x10FFFF) {
94 throw std::runtime_error(
95 to<std::string>("folly::decodeUtf8 i=", i, " d=", d));
104 throw std::runtime_error("folly::decodeUtf8 encoding length maxed out");
107 // Escape a string so that it is legal to print it in JSON text.
108 void escapeString(StringPiece input,
110 const serialization_opts& opts) {
111 auto hexDigit = [] (int c) -> char {
112 return c < 10 ? c + '0' : c - 10 + 'a';
115 out.reserve(out.size() + input.size() + 2);
118 auto* p = reinterpret_cast<const unsigned char*>(input.begin());
119 auto* q = reinterpret_cast<const unsigned char*>(input.begin());
120 auto* e = reinterpret_cast<const unsigned char*>(input.end());
123 // Since non-ascii encoding inherently does utf8 validation
124 // we explicitly validate utf8 only if non-ascii encoding is disabled.
125 if (opts.validate_utf8 && !opts.encode_non_ascii) {
126 // to achieve better spatial and temporal coherence
127 // we do utf8 validation progressively along with the
128 // string-escaping instead of two separate passes
130 // as the encoding progresses, q will stay at or ahead of p
133 // as p catches up with q, move q forward
135 // calling utf8_decode has the side effect of
136 // checking that utf8 encodings are valid
141 if (opts.encode_non_ascii && (*p & 0x80)) {
142 // note that this if condition captures utf8 chars
143 // with value > 127, so size > 1 byte
144 char32_t v = decodeUtf8(p, e);
146 out.push_back(hexDigit(v >> 12));
147 out.push_back(hexDigit((v >> 8) & 0x0f));
148 out.push_back(hexDigit((v >> 4) & 0x0f));
149 out.push_back(hexDigit(v & 0x0f));
150 } else if (*p == '\\' || *p == '\"') {
153 } else if (*p <= 0x1f) {
155 case '\b': out.append("\\b"); p++; break;
156 case '\f': out.append("\\f"); p++; break;
157 case '\n': out.append("\\n"); p++; break;
158 case '\r': out.append("\\r"); p++; break;
159 case '\t': out.append("\\t"); p++; break;
161 // note that this if condition captures non readable chars
162 // with value < 32, so size = 1 byte (e.g control chars).
164 out.push_back(hexDigit((*p & 0xf0) >> 4));
165 out.push_back(hexDigit(*p & 0xf));
177 explicit Printer(fbstring& out,
178 unsigned* indentLevel,
179 serialization_opts const* opts)
181 , indentLevel_(indentLevel)
185 void operator()(dynamic const& v) const {
187 case dynamic::DOUBLE:
188 toAppend(v.asDouble(), &out_);
190 case dynamic::INT64: {
191 auto intval = v.asInt();
192 if (opts_.javascript_safe) {
193 // Use folly::to to check that this integer can be represented
194 // as a double without loss of precision.
195 intval = int64_t(to<double>(intval));
197 toAppend(intval, &out_);
201 out_ += v.asBool() ? "true" : "false";
206 case dynamic::STRING:
207 escapeString(v.asString(), out_, opts_);
209 case dynamic::OBJECT:
216 CHECK(0) << "Bad type " << v.type();
221 void printKV(const std::pair<dynamic, dynamic>& p) const {
222 if (!opts_.allow_non_string_keys && !p.first.isString()) {
223 throw std::runtime_error("folly::toJson: JSON object key was not a "
231 void printObject(dynamic const& o) const {
240 auto it = o.items().begin();
242 for (++it; it != o.items().end(); ++it) {
252 void printArray(dynamic const& a) const {
262 for (auto& val : makeRange(boost::next(a.begin()), a.end())) {
273 void outdent() const {
279 void indent() const {
285 void newline() const {
287 out_ += to<fbstring>('\n', fbstring(*indentLevel_ * 2, ' '));
291 void mapColon() const {
292 out_ += indentLevel_ ? " : " : ":";
297 unsigned* const indentLevel_;
298 serialization_opts const& opts_;
301 //////////////////////////////////////////////////////////////////////
303 struct ParseError : std::runtime_error {
304 explicit ParseError(int line)
305 : std::runtime_error(to<std::string>("json parse error on line ", line))
308 explicit ParseError(int line, std::string const& context,
309 std::string const& expected)
310 : std::runtime_error(to<std::string>("json parse error on line ", line,
311 !context.empty() ? to<std::string>(" near `", context, '\'')
316 explicit ParseError(std::string const& what)
317 : std::runtime_error("json parse error: " + what)
321 // Wraps our input buffer with some helper functions.
323 explicit Input(StringPiece range)
330 Input(Input const&) = delete;
331 Input& operator=(Input const&) = delete;
333 char const* begin() const { return range_.begin(); }
335 // Parse ahead for as long as the supplied predicate is satisfied,
336 // returning a range of what was skipped.
337 template<class Predicate>
338 StringPiece skipWhile(const Predicate& p) {
339 std::size_t skipped = 0;
340 for (; skipped < range_.size(); ++skipped) {
341 if (!p(range_[skipped])) {
344 if (range_[skipped] == '\n') {
348 auto ret = range_.subpiece(0, skipped);
349 range_.advance(skipped);
354 StringPiece skipDigits() {
355 return skipWhile([] (char c) { return c >= '0' && c <= '9'; });
358 void skipWhitespace() {
359 // Spaces other than ' ' characters are less common but should be
360 // checked. This configuration where we loop on the ' '
361 // separately from oddspaces was empirically fastest.
362 auto oddspace = [] (char c) {
363 return c == '\n' || c == '\t' || c == '\r';
367 for (; !range_.empty() && range_.front() == ' '; range_.pop_front()) {
369 if (!range_.empty() && oddspace(range_.front())) {
376 void expect(char c) {
378 throw ParseError(lineNum_, context(),
379 to<std::string>("expected '", c, '\''));
384 std::size_t size() const {
385 return range_.size();
388 int operator*() const {
400 return to<T>(&range_);
401 } catch (std::exception const& e) {
406 bool consume(StringPiece str) {
407 if (boost::starts_with(range_, str)) {
408 range_.advance(str.size());
415 std::string context() const {
416 return range_.subpiece(0, 16 /* arbitrary */).toString();
419 dynamic error(char const* what) const {
420 throw ParseError(lineNum_, context(), what);
424 void storeCurrent() {
425 current_ = range_.empty() ? EOF : range_.front();
434 dynamic parseValue(Input& in);
435 fbstring parseString(Input& in);
437 dynamic parseObject(Input& in) {
441 dynamic ret = dynamic::object;
451 in.error("expected string for object key name");
453 auto key = parseString(in);
457 ret.insert(std::move(key), parseValue(in));
470 dynamic parseArray(Input& in) {
483 ret.push_back(parseValue(in));
496 dynamic parseNumber(Input& in) {
497 bool const negative = (*in == '-');
500 if (in.consume("Infinity")) {
501 return -std::numeric_limits<double>::infinity();
505 auto integral = in.skipDigits();
506 if (integral.empty()) {
507 in.error("expected digits after `-'");
509 auto const wasE = *in == 'e' || *in == 'E';
510 if (*in != '.' && !wasE) {
511 auto val = to<int64_t>(integral);
519 auto end = !wasE ? (++in, in.skipDigits().end()) : in.begin();
520 if (*in == 'e' || *in == 'E') {
522 if (*in == '+' || *in == '-') {
525 auto expPart = in.skipDigits();
528 auto fullNum = makeRange(integral.begin(), end);
530 auto val = to<double>(fullNum);
537 fbstring decodeUnicodeEscape(Input& in) {
538 auto hexVal = [&] (char c) -> unsigned {
539 return c >= '0' && c <= '9' ? c - '0' :
540 c >= 'a' && c <= 'f' ? c - 'a' + 10 :
541 c >= 'A' && c <= 'F' ? c - 'A' + 10 :
542 (in.error("invalid hex digit"), 0);
545 auto readHex = [&]() -> uint16_t {
547 in.error("expected 4 hex digits");
550 uint16_t ret = hexVal(*in) * 4096;
552 ret += hexVal(*in) * 256;
554 ret += hexVal(*in) * 16;
562 * If the value encoded is in the surrogate pair range, we need to
563 * make sure there is another escape that we can use also.
565 uint32_t codePoint = readHex();
566 if (codePoint >= 0xd800 && codePoint <= 0xdbff) {
567 if (!in.consume("\\u")) {
568 in.error("expected another unicode escape for second half of "
571 uint16_t second = readHex();
572 if (second >= 0xdc00 && second <= 0xdfff) {
573 codePoint = 0x10000 + ((codePoint & 0x3ff) << 10) +
576 in.error("second character in surrogate pair is invalid");
578 } else if (codePoint >= 0xdc00 && codePoint <= 0xdfff) {
579 in.error("invalid unicode code point (in range [0xdc00,0xdfff])");
582 return codePointToUtf8(codePoint);
585 fbstring parseString(Input& in) {
591 auto range = in.skipWhile(
592 [] (char c) { return c != '\"' && c != '\\'; }
594 ret.append(range.begin(), range.end());
603 case '\"': ret.push_back('\"'); ++in; break;
604 case '\\': ret.push_back('\\'); ++in; break;
605 case '/': ret.push_back('/'); ++in; break;
606 case 'b': ret.push_back('\b'); ++in; break;
607 case 'f': ret.push_back('\f'); ++in; break;
608 case 'n': ret.push_back('\n'); ++in; break;
609 case 'r': ret.push_back('\r'); ++in; break;
610 case 't': ret.push_back('\t'); ++in; break;
611 case 'u': ++in; ret += decodeUnicodeEscape(in); break;
612 default: in.error(to<fbstring>("unknown escape ", *in,
613 " in string").c_str());
618 in.error("unterminated string");
622 * Apparently we're actually supposed to ban all control
623 * characters from strings. This seems unnecessarily
624 * restrictive, so we're only banning zero bytes. (Since the
625 * string is presumed to be UTF-8 encoded it's fine to just
628 in.error("null byte in string");
638 dynamic parseValue(Input& in) {
640 return *in == '[' ? parseArray(in) :
641 *in == '{' ? parseObject(in) :
642 *in == '\"' ? parseString(in) :
643 (*in == '-' || (*in >= '0' && *in <= '9')) ? parseNumber(in) :
644 in.consume("true") ? true :
645 in.consume("false") ? false :
646 in.consume("null") ? nullptr :
647 in.consume("Infinity") ? std::numeric_limits<double>::infinity() :
648 in.consume("NaN") ? std::numeric_limits<double>::quiet_NaN() :
649 in.error("expected json value");
654 //////////////////////////////////////////////////////////////////////
656 fbstring serialize(dynamic const& dyn, serialization_opts const& opts) {
658 unsigned indentLevel = 0;
659 Printer p(ret, opts.pretty_formatting ? &indentLevel : nullptr, &opts);
666 //////////////////////////////////////////////////////////////////////
668 dynamic parseJson(StringPiece range) {
669 json::Input in(range);
671 auto ret = parseValue(in);
673 if (*in != '\0' && in.size()) {
674 in.error("parsing didn't consume all input");
679 fbstring toJson(dynamic const& dyn) {
680 return json::serialize(dyn, json::serialization_opts());
683 fbstring toPrettyJson(dynamic const& dyn) {
684 json::serialization_opts opts;
685 opts.pretty_formatting = true;
686 return json::serialize(dyn, opts);
689 //////////////////////////////////////////////////////////////////////
690 // dynamic::print_as_pseudo_json() is implemented here for header
691 // ordering reasons (most of the dynamic implementation is in
692 // dynamic-inl.h, which we don't want to include json.h).
694 void dynamic::print_as_pseudo_json(std::ostream& out) const {
695 json::serialization_opts opts;
696 opts.allow_non_string_keys = true;
697 out << json::serialize(*this, opts);
700 //////////////////////////////////////////////////////////////////////