2 * Copyright 2017-present Facebook, Inc.
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
16 #include <folly/json.h>
20 #include <type_traits>
22 #include <boost/algorithm/string.hpp>
23 #include <boost/next_prior.hpp>
25 #include <folly/Conv.h>
26 #include <folly/Portability.h>
27 #include <folly/Range.h>
28 #include <folly/String.h>
29 #include <folly/Unicode.h>
30 #include <folly/lang/Bits.h>
31 #include <folly/portability/Constexpr.h>
35 //////////////////////////////////////////////////////////////////////
43 unsigned* indentLevel,
44 serialization_opts const* opts)
45 : out_(out), indentLevel_(indentLevel), opts_(*opts) {}
47 void operator()(dynamic const& v) const {
50 if (!opts_.allow_nan_inf &&
51 (std::isnan(v.asDouble()) || std::isinf(v.asDouble()))) {
52 throw std::runtime_error("folly::toJson: JSON object value was a "
55 toAppend(v.asDouble(), &out_, opts_.double_mode, opts_.double_num_digits);
57 case dynamic::INT64: {
58 auto intval = v.asInt();
59 if (opts_.javascript_safe) {
60 // Use folly::to to check that this integer can be represented
61 // as a double without loss of precision.
62 intval = int64_t(to<double>(intval));
64 toAppend(intval, &out_);
68 out_ += v.asBool() ? "true" : "false";
74 escapeString(v.asString(), out_, opts_);
83 CHECK(0) << "Bad type " << v.type();
88 void printKV(const std::pair<const dynamic, dynamic>& p) const {
89 if (!opts_.allow_non_string_keys && !p.first.isString()) {
90 throw std::runtime_error("folly::toJson: JSON object key was not a "
98 template <typename Iterator>
99 void printKVPairs(Iterator begin, Iterator end) const {
101 for (++begin; begin != end; ++begin) {
108 void printObject(dynamic const& o) const {
117 if (opts_.sort_keys || opts_.sort_keys_by) {
118 using ref = std::reference_wrapper<decltype(o.items())::value_type const>;
119 std::vector<ref> refs(o.items().begin(), o.items().end());
121 using SortByRef = FunctionRef<bool(dynamic const&, dynamic const&)>;
122 auto const& sort_keys_by = opts_.sort_keys_by
123 ? SortByRef(opts_.sort_keys_by)
124 : SortByRef(std::less<dynamic>());
125 std::sort(refs.begin(), refs.end(), [&](ref a, ref b) {
126 // Only compare keys. No ordering among identical keys.
127 return sort_keys_by(a.get().first, b.get().first);
129 printKVPairs(refs.cbegin(), refs.cend());
131 printKVPairs(o.items().begin(), o.items().end());
138 void printArray(dynamic const& a) const {
148 for (auto& val : range(boost::next(a.begin()), a.end())) {
159 void outdent() const {
165 void indent() const {
171 void newline() const {
173 out_ += to<std::string>('\n', std::string(*indentLevel_ * 2, ' '));
177 void mapColon() const {
178 out_ += indentLevel_ ? " : " : ":";
183 unsigned* const indentLevel_;
184 serialization_opts const& opts_;
187 //////////////////////////////////////////////////////////////////////
189 struct FOLLY_EXPORT ParseError : std::runtime_error {
192 std::string const& context,
193 std::string const& expected)
194 : std::runtime_error(to<std::string>(
195 "json parse error on line ",
197 !context.empty() ? to<std::string>(" near `", context, '\'') : "",
202 // Wraps our input buffer with some helper functions.
204 explicit Input(StringPiece range, json::serialization_opts const* opts)
212 Input(Input const&) = delete;
213 Input& operator=(Input const&) = delete;
215 char const* begin() const { return range_.begin(); }
217 // Parse ahead for as long as the supplied predicate is satisfied,
218 // returning a range of what was skipped.
219 template <class Predicate>
220 StringPiece skipWhile(const Predicate& p) {
221 std::size_t skipped = 0;
222 for (; skipped < range_.size(); ++skipped) {
223 if (!p(range_[skipped])) {
226 if (range_[skipped] == '\n') {
230 auto ret = range_.subpiece(0, skipped);
231 range_.advance(skipped);
236 StringPiece skipDigits() {
237 return skipWhile([] (char c) { return c >= '0' && c <= '9'; });
240 StringPiece skipMinusAndDigits() {
241 bool firstChar = true;
242 return skipWhile([&firstChar] (char c) {
243 bool result = (c >= '0' && c <= '9') || (firstChar && c == '-');
249 void skipWhitespace() {
250 range_ = folly::skipWhitespace(range_);
254 void expect(char c) {
256 throw ParseError(lineNum_, context(),
257 to<std::string>("expected '", c, '\''));
262 std::size_t size() const {
263 return range_.size();
266 int operator*() const {
278 return to<T>(&range_);
279 } catch (std::exception const& e) {
284 bool consume(StringPiece str) {
285 if (boost::starts_with(range_, str)) {
286 range_.advance(str.size());
293 std::string context() const {
294 return range_.subpiece(0, 16 /* arbitrary */).toString();
297 dynamic error(char const* what) const {
298 throw ParseError(lineNum_, context(), what);
301 json::serialization_opts const& getOpts() {
305 void incrementRecursionLevel() {
306 if (currentRecursionLevel_ > opts_.recursion_limit) {
307 error("recursion limit exceeded");
309 currentRecursionLevel_++;
312 void decrementRecursionLevel() {
313 currentRecursionLevel_--;
317 void storeCurrent() {
318 current_ = range_.empty() ? EOF : range_.front();
323 json::serialization_opts const& opts_;
326 unsigned int currentRecursionLevel_{0};
329 class RecursionGuard {
331 explicit RecursionGuard(Input& in) : in_(in) {
332 in_.incrementRecursionLevel();
336 in_.decrementRecursionLevel();
343 dynamic parseValue(Input& in);
344 std::string parseString(Input& in);
345 dynamic parseNumber(Input& in);
347 dynamic parseObject(Input& in) {
351 dynamic ret = dynamic::object;
360 if (in.getOpts().allow_trailing_comma && *in == '}') {
363 if (*in == '\"') { // string
364 auto key = parseString(in);
368 ret.insert(std::move(key), parseValue(in));
369 } else if (!in.getOpts().allow_non_string_keys) {
370 in.error("expected string for object key name");
372 auto key = parseValue(in);
376 ret.insert(std::move(key), parseValue(in));
391 dynamic parseArray(Input& in) {
395 dynamic ret = dynamic::array;
404 if (in.getOpts().allow_trailing_comma && *in == ']') {
407 ret.push_back(parseValue(in));
420 dynamic parseNumber(Input& in) {
421 bool const negative = (*in == '-');
422 if (negative && in.consume("-Infinity")) {
423 if (in.getOpts().parse_numbers_as_strings) {
426 return -std::numeric_limits<double>::infinity();
430 auto integral = in.skipMinusAndDigits();
431 if (negative && integral.size() < 2) {
432 in.error("expected digits after `-'");
435 auto const wasE = *in == 'e' || *in == 'E';
437 constexpr const char* maxInt = "9223372036854775807";
438 constexpr const char* minInt = "-9223372036854775808";
439 constexpr auto maxIntLen = constexpr_strlen(maxInt);
440 constexpr auto minIntLen = constexpr_strlen(minInt);
442 if (*in != '.' && !wasE && in.getOpts().parse_numbers_as_strings) {
446 if (*in != '.' && !wasE) {
447 if (LIKELY(!in.getOpts().double_fallback || integral.size() < maxIntLen) ||
448 (!negative && integral.size() == maxIntLen && integral <= maxInt) ||
449 (negative && integral.size() == minIntLen && integral <= minInt)) {
450 auto val = to<int64_t>(integral);
454 auto val = to<double>(integral);
460 auto end = !wasE ? (++in, in.skipDigits().end()) : in.begin();
461 if (*in == 'e' || *in == 'E') {
463 if (*in == '+' || *in == '-') {
466 auto expPart = in.skipDigits();
469 auto fullNum = range(integral.begin(), end);
470 if (in.getOpts().parse_numbers_as_strings) {
473 auto val = to<double>(fullNum);
477 std::string decodeUnicodeEscape(Input& in) {
478 auto hexVal = [&] (int c) -> uint16_t {
480 c >= '0' && c <= '9' ? c - '0' :
481 c >= 'a' && c <= 'f' ? c - 'a' + 10 :
482 c >= 'A' && c <= 'F' ? c - 'A' + 10 :
483 (in.error("invalid hex digit"), 0));
486 auto readHex = [&]() -> uint16_t {
488 in.error("expected 4 hex digits");
491 uint16_t ret = uint16_t(hexVal(*in) * 4096);
493 ret += hexVal(*in) * 256;
495 ret += hexVal(*in) * 16;
503 * If the value encoded is in the surrogate pair range, we need to
504 * make sure there is another escape that we can use also.
506 uint32_t codePoint = readHex();
507 if (codePoint >= 0xd800 && codePoint <= 0xdbff) {
508 if (!in.consume("\\u")) {
509 in.error("expected another unicode escape for second half of "
512 uint16_t second = readHex();
513 if (second >= 0xdc00 && second <= 0xdfff) {
514 codePoint = 0x10000 + ((codePoint & 0x3ff) << 10) +
517 in.error("second character in surrogate pair is invalid");
519 } else if (codePoint >= 0xdc00 && codePoint <= 0xdfff) {
520 in.error("invalid unicode code point (in range [0xdc00,0xdfff])");
523 return codePointToUtf8(codePoint);
526 std::string parseString(Input& in) {
527 DCHECK_EQ(*in, '\"');
532 auto range = in.skipWhile(
533 [] (char c) { return c != '\"' && c != '\\'; }
535 ret.append(range.begin(), range.end());
544 case '\"': ret.push_back('\"'); ++in; break;
545 case '\\': ret.push_back('\\'); ++in; break;
546 case '/': ret.push_back('/'); ++in; break;
547 case 'b': ret.push_back('\b'); ++in; break;
548 case 'f': ret.push_back('\f'); ++in; break;
549 case 'n': ret.push_back('\n'); ++in; break;
550 case 'r': ret.push_back('\r'); ++in; break;
551 case 't': ret.push_back('\t'); ++in; break;
552 case 'u': ++in; ret += decodeUnicodeEscape(in); break;
554 in.error(to<std::string>("unknown escape ", *in, " in string").c_str());
559 in.error("unterminated string");
563 * Apparently we're actually supposed to ban all control
564 * characters from strings. This seems unnecessarily
565 * restrictive, so we're only banning zero bytes. (Since the
566 * string is presumed to be UTF-8 encoded it's fine to just
569 in.error("null byte in string");
572 ret.push_back(char(*in));
579 dynamic parseValue(Input& in) {
580 RecursionGuard guard(in);
583 return *in == '[' ? parseArray(in) :
584 *in == '{' ? parseObject(in) :
585 *in == '\"' ? parseString(in) :
586 (*in == '-' || (*in >= '0' && *in <= '9')) ? parseNumber(in) :
587 in.consume("true") ? true :
588 in.consume("false") ? false :
589 in.consume("null") ? nullptr :
590 in.consume("Infinity") ?
591 (in.getOpts().parse_numbers_as_strings ? (dynamic)"Infinity" :
592 (dynamic)std::numeric_limits<double>::infinity()) :
594 (in.getOpts().parse_numbers_as_strings ? (dynamic)"NaN" :
595 (dynamic)std::numeric_limits<double>::quiet_NaN()) :
596 in.error("expected json value");
601 //////////////////////////////////////////////////////////////////////
603 std::string serialize(dynamic const& dyn, serialization_opts const& opts) {
605 unsigned indentLevel = 0;
606 Printer p(ret, opts.pretty_formatting ? &indentLevel : nullptr, &opts);
611 // Fast path to determine the longest prefix that can be left
612 // unescaped in a string of sizeof(T) bytes packed in an integer of
615 size_t firstEscapableInWord(T s) {
616 static_assert(std::is_unsigned<T>::value, "Unsigned integer required");
617 static constexpr T kOnes = ~T() / 255; // 0x...0101
618 static constexpr T kMsbs = kOnes * 0x80; // 0x...8080
620 // Sets the MSB of bytes < b. Precondition: b < 128.
621 auto isLess = [](T w, uint8_t b) {
622 // A byte is < b iff subtracting b underflows, so we check that
623 // the MSB wasn't set before and it's set after the subtraction.
624 return (w - kOnes * b) & ~w & kMsbs;
627 auto isChar = [&](uint8_t c) {
628 // A byte is == c iff it is 0 if xored with c.
629 return isLess(s ^ (kOnes * c), 1);
632 // The following masks have the MSB set for each byte of the word
633 // that satisfies the corresponding condition.
634 auto isHigh = s & kMsbs; // >= 128
635 auto isLow = isLess(s, 0x20); // <= 0x1f
636 auto needsEscape = isHigh | isLow | isChar('\\') | isChar('"');
642 if (folly::kIsLittleEndian) {
643 return folly::findFirstSet(needsEscape) / 8 - 1;
645 return sizeof(T) - folly::findLastSet(needsEscape) / 8;
649 // Escape a string so that it is legal to print it in JSON text.
653 const serialization_opts& opts) {
654 auto hexDigit = [] (uint8_t c) -> char {
655 return c < 10 ? c + '0' : c - 10 + 'a';
660 auto* p = reinterpret_cast<const unsigned char*>(input.begin());
661 auto* q = reinterpret_cast<const unsigned char*>(input.begin());
662 auto* e = reinterpret_cast<const unsigned char*>(input.end());
665 // Find the longest prefix that does not need escaping, and copy
666 // it literally into the output string.
668 while (firstEsc < e) {
669 auto avail = e - firstEsc;
672 word = folly::loadUnaligned<uint64_t>(firstEsc);
674 memcpy(static_cast<void*>(&word), firstEsc, avail);
676 auto prefix = firstEscapableInWord(word);
677 DCHECK_LE(prefix, avail);
684 out.append(reinterpret_cast<const char*>(p), firstEsc - p);
686 // We can't be in the middle of a multibyte sequence, so we can reset q.
693 // Handle the next byte that may need escaping.
695 // Since non-ascii encoding inherently does utf8 validation
696 // we explicitly validate utf8 only if non-ascii encoding is disabled.
697 if ((opts.validate_utf8 || opts.skip_invalid_utf8)
698 && !opts.encode_non_ascii) {
699 // To achieve better spatial and temporal coherence
700 // we do utf8 validation progressively along with the
701 // string-escaping instead of two separate passes.
703 // As the encoding progresses, q will stay at or ahead of p.
706 // As p catches up with q, move q forward.
708 // calling utf8_decode has the side effect of
709 // checking that utf8 encodings are valid
710 char32_t v = utf8ToCodePoint(q, e, opts.skip_invalid_utf8);
711 if (opts.skip_invalid_utf8 && v == U'\ufffd') {
712 out.append(u8"\ufffd");
718 if (opts.encode_non_ascii && (*p & 0x80)) {
719 // note that this if condition captures utf8 chars
720 // with value > 127, so size > 1 byte
721 char32_t v = utf8ToCodePoint(p, e, opts.skip_invalid_utf8);
722 char buf[] = "\\u\0\0\0\0";
723 buf[2] = hexDigit(uint8_t(v >> 12));
724 buf[3] = hexDigit((v >> 8) & 0x0f);
725 buf[4] = hexDigit((v >> 4) & 0x0f);
726 buf[5] = hexDigit(v & 0x0f);
728 } else if (*p == '\\' || *p == '\"') {
732 } else if (*p <= 0x1f) {
734 case '\b': out.append("\\b"); p++; break;
735 case '\f': out.append("\\f"); p++; break;
736 case '\n': out.append("\\n"); p++; break;
737 case '\r': out.append("\\r"); p++; break;
738 case '\t': out.append("\\t"); p++; break;
740 // Note that this if condition captures non readable chars
741 // with value < 32, so size = 1 byte (e.g control chars).
742 char buf[] = "\\u00\0\0";
743 buf[4] = hexDigit(uint8_t((*p & 0xf0) >> 4));
744 buf[5] = hexDigit(uint8_t(*p & 0xf));
749 out.push_back(char(*p++));
756 std::string stripComments(StringPiece jsonC) {
763 } state = State::None;
765 for (size_t i = 0; i < jsonC.size(); ++i) {
766 auto s = jsonC.subpiece(i);
769 if (s.startsWith("/*")) {
770 state = State::InlineComment;
773 } else if (s.startsWith("//")) {
774 state = State::LineComment;
777 } else if (s[0] == '\"') {
778 state = State::InString;
780 result.push_back(s[0]);
782 case State::InString:
784 if (UNLIKELY(s.size() == 1)) {
785 throw std::logic_error("Invalid JSONC: string is not terminated");
787 result.push_back(s[0]);
788 result.push_back(s[1]);
791 } else if (s[0] == '\"') {
794 result.push_back(s[0]);
796 case State::InlineComment:
797 if (s.startsWith("*/")) {
802 case State::LineComment:
804 // skip the line break. It doesn't matter.
809 throw std::logic_error("Unknown comment state");
817 //////////////////////////////////////////////////////////////////////
819 dynamic parseJson(StringPiece range) {
820 return parseJson(range, json::serialization_opts());
825 json::serialization_opts const& opts) {
827 json::Input in(range, &opts);
829 auto ret = parseValue(in);
831 if (in.size() && *in != '\0') {
832 in.error("parsing didn't consume all input");
837 std::string toJson(dynamic const& dyn) {
838 return json::serialize(dyn, json::serialization_opts());
841 std::string toPrettyJson(dynamic const& dyn) {
842 json::serialization_opts opts;
843 opts.pretty_formatting = true;
844 return json::serialize(dyn, opts);
847 //////////////////////////////////////////////////////////////////////
848 // dynamic::print_as_pseudo_json() is implemented here for header
849 // ordering reasons (most of the dynamic implementation is in
850 // dynamic-inl.h, which we don't want to include json.h).
852 void dynamic::print_as_pseudo_json(std::ostream& out) const {
853 json::serialization_opts opts;
854 opts.allow_non_string_keys = true;
855 opts.allow_nan_inf = true;
856 out << json::serialize(*this, opts);
859 void PrintTo(const dynamic& dyn, std::ostream* os) {
860 json::serialization_opts opts;
861 opts.allow_nan_inf = true;
862 opts.allow_non_string_keys = true;
863 opts.pretty_formatting = true;
864 opts.sort_keys = true;
865 *os << json::serialize(dyn, opts);
868 //////////////////////////////////////////////////////////////////////