2 * Copyright 2016 Facebook, Inc.
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
17 #include <folly/json.h>
19 #include <boost/next_prior.hpp>
20 #include <boost/algorithm/string.hpp>
22 #include <folly/Conv.h>
23 #include <folly/Portability.h>
24 #include <folly/Range.h>
25 #include <folly/String.h>
26 #include <folly/Unicode.h>
27 #include <folly/portability/Constexpr.h>
31 //////////////////////////////////////////////////////////////////////
37 const unsigned char*& p,
38 const unsigned char* const e,
40 /* The following encodings are valid, except for the 5 and 6 byte
44 * 1110xxxx 10xxxxxx 10xxxxxx
45 * 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
46 * 111110xx 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx
47 * 1111110x 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx
50 auto skip = [&] { ++p; return U'\ufffd'; };
53 if (skipOnError) return skip();
54 throw std::runtime_error("folly::decodeUtf8 empty/invalid string");
57 unsigned char fst = *p;
63 static const uint32_t bitMask[] = {
70 // upper control bits are masked out later
73 if ((fst & 0xC0) != 0xC0) {
74 if (skipOnError) return skip();
75 throw std::runtime_error(to<std::string>("folly::decodeUtf8 i=0 d=", d));
80 for (unsigned int i = 1; i != 3 && p + i < e; ++i) {
81 unsigned char tmp = p[i];
83 if ((tmp & 0xC0) != 0x80) {
84 if (skipOnError) return skip();
85 throw std::runtime_error(
86 to<std::string>("folly::decodeUtf8 i=", i, " tmp=", (uint32_t)tmp));
89 d = (d << 6) | (tmp & 0x3F);
95 // overlong, could have been encoded with i bytes
96 if ((d & ~bitMask[i - 1]) == 0) {
97 if (skipOnError) return skip();
98 throw std::runtime_error(
99 to<std::string>("folly::decodeUtf8 i=", i, " d=", d));
102 // check for surrogates only needed for 3 bytes
104 if ((d >= 0xD800 && d <= 0xDFFF) || d > 0x10FFFF) {
105 if (skipOnError) return skip();
106 throw std::runtime_error(
107 to<std::string>("folly::decodeUtf8 i=", i, " d=", d));
116 if (skipOnError) return skip();
117 throw std::runtime_error("folly::decodeUtf8 encoding length maxed out");
121 explicit Printer(fbstring& out,
122 unsigned* indentLevel,
123 serialization_opts const* opts)
125 , indentLevel_(indentLevel)
129 void operator()(dynamic const& v) const {
131 case dynamic::DOUBLE:
132 if (!opts_.allow_nan_inf &&
133 (std::isnan(v.asDouble()) || std::isinf(v.asDouble()))) {
134 throw std::runtime_error("folly::toJson: JSON object value was a "
137 toAppend(v.asDouble(), &out_, opts_.double_mode, opts_.double_num_digits);
139 case dynamic::INT64: {
140 auto intval = v.asInt();
141 if (opts_.javascript_safe) {
142 // Use folly::to to check that this integer can be represented
143 // as a double without loss of precision.
144 intval = int64_t(to<double>(intval));
146 toAppend(intval, &out_);
150 out_ += v.asBool() ? "true" : "false";
155 case dynamic::STRING:
156 escapeString(v.asString(), out_, opts_);
158 case dynamic::OBJECT:
165 CHECK(0) << "Bad type " << v.type();
170 void printKV(const std::pair<const dynamic, dynamic>& p) const {
171 if (!opts_.allow_non_string_keys && !p.first.isString()) {
172 throw std::runtime_error("folly::toJson: JSON object key was not a "
180 template <typename Iterator>
181 void printKVPairs(Iterator begin, Iterator end) const {
183 for (++begin; begin != end; ++begin) {
190 void printObject(dynamic const& o) const {
199 if (opts_.sort_keys) {
200 std::vector<std::pair<dynamic, dynamic>> items(
201 o.items().begin(), o.items().end());
202 std::sort(items.begin(), items.end());
203 printKVPairs(items.begin(), items.end());
205 printKVPairs(o.items().begin(), o.items().end());
212 void printArray(dynamic const& a) const {
222 for (auto& val : range(boost::next(a.begin()), a.end())) {
233 void outdent() const {
239 void indent() const {
245 void newline() const {
247 out_ += to<fbstring>('\n', fbstring(*indentLevel_ * 2, ' '));
251 void mapColon() const {
252 out_ += indentLevel_ ? " : " : ":";
257 unsigned* const indentLevel_;
258 serialization_opts const& opts_;
261 //////////////////////////////////////////////////////////////////////
263 struct ParseError : std::runtime_error {
264 explicit ParseError(int line)
265 : std::runtime_error(to<std::string>("json parse error on line ", line))
268 explicit ParseError(int line, std::string const& context,
269 std::string const& expected)
270 : std::runtime_error(to<std::string>("json parse error on line ", line,
271 !context.empty() ? to<std::string>(" near `", context, '\'')
276 explicit ParseError(std::string const& msg)
277 : std::runtime_error("json parse error: " + msg)
281 // Wraps our input buffer with some helper functions.
283 explicit Input(StringPiece range, json::serialization_opts const* opts)
291 Input(Input const&) = delete;
292 Input& operator=(Input const&) = delete;
294 char const* begin() const { return range_.begin(); }
296 // Parse ahead for as long as the supplied predicate is satisfied,
297 // returning a range of what was skipped.
298 template<class Predicate>
299 StringPiece skipWhile(const Predicate& p) {
300 std::size_t skipped = 0;
301 for (; skipped < range_.size(); ++skipped) {
302 if (!p(range_[skipped])) {
305 if (range_[skipped] == '\n') {
309 auto ret = range_.subpiece(0, skipped);
310 range_.advance(skipped);
315 StringPiece skipDigits() {
316 return skipWhile([] (char c) { return c >= '0' && c <= '9'; });
319 StringPiece skipMinusAndDigits() {
320 bool firstChar = true;
321 return skipWhile([&firstChar] (char c) {
322 bool result = (c >= '0' && c <= '9') || (firstChar && c == '-');
328 void skipWhitespace() {
329 range_ = folly::skipWhitespace(range_);
333 void expect(char c) {
335 throw ParseError(lineNum_, context(),
336 to<std::string>("expected '", c, '\''));
341 std::size_t size() const {
342 return range_.size();
345 int operator*() const {
357 return to<T>(&range_);
358 } catch (std::exception const& e) {
363 bool consume(StringPiece str) {
364 if (boost::starts_with(range_, str)) {
365 range_.advance(str.size());
372 std::string context() const {
373 return range_.subpiece(0, 16 /* arbitrary */).toString();
376 dynamic error(char const* what) const {
377 throw ParseError(lineNum_, context(), what);
380 json::serialization_opts const& getOpts() {
385 void storeCurrent() {
386 current_ = range_.empty() ? EOF : range_.front();
391 json::serialization_opts const& opts_;
396 dynamic parseValue(Input& in);
397 fbstring parseString(Input& in);
398 dynamic parseNumber(Input& in);
400 dynamic parseObject(Input& in) {
404 dynamic ret = dynamic::object;
413 if (in.getOpts().allow_trailing_comma && *in == '}') {
416 if (*in == '\"') { // string
417 auto key = parseString(in);
421 ret.insert(std::move(key), parseValue(in));
422 } else if (!in.getOpts().allow_non_string_keys) {
423 in.error("expected string for object key name");
425 auto key = parseValue(in);
429 ret.insert(std::move(key), parseValue(in));
444 dynamic parseArray(Input& in) {
448 dynamic ret = dynamic::array;
457 if (in.getOpts().allow_trailing_comma && *in == ']') {
460 ret.push_back(parseValue(in));
473 dynamic parseNumber(Input& in) {
474 bool const negative = (*in == '-');
475 if (negative && in.consume("-Infinity")) {
476 if (in.getOpts().parse_numbers_as_strings) {
479 return -std::numeric_limits<double>::infinity();
483 auto integral = in.skipMinusAndDigits();
484 if (negative && integral.size() < 2) {
485 in.error("expected digits after `-'");
488 auto const wasE = *in == 'e' || *in == 'E';
490 constexpr const char* maxInt = "9223372036854775807";
491 constexpr const char* minInt = "9223372036854775808";
492 constexpr auto maxIntLen = constexpr_strlen(maxInt);
495 if (*in != '.' && !wasE && in.getOpts().parse_numbers_as_strings) {
499 if (*in != '.' && !wasE) {
500 if (LIKELY(!in.getOpts().double_fallback || integral.size() < maxIntLen) ||
501 (integral.size() == maxIntLen &&
502 (integral <= maxInt || (integral == minInt && negative)))) {
503 auto val = to<int64_t>(integral);
507 auto val = to<double>(integral);
513 auto end = !wasE ? (++in, in.skipDigits().end()) : in.begin();
514 if (*in == 'e' || *in == 'E') {
516 if (*in == '+' || *in == '-') {
519 auto expPart = in.skipDigits();
522 auto fullNum = range(integral.begin(), end);
523 if (in.getOpts().parse_numbers_as_strings) {
526 auto val = to<double>(fullNum);
530 fbstring decodeUnicodeEscape(Input& in) {
531 auto hexVal = [&] (char c) -> unsigned {
532 return c >= '0' && c <= '9' ? c - '0' :
533 c >= 'a' && c <= 'f' ? c - 'a' + 10 :
534 c >= 'A' && c <= 'F' ? c - 'A' + 10 :
535 (in.error("invalid hex digit"), 0);
538 auto readHex = [&]() -> uint16_t {
540 in.error("expected 4 hex digits");
543 uint16_t ret = hexVal(*in) * 4096;
545 ret += hexVal(*in) * 256;
547 ret += hexVal(*in) * 16;
555 * If the value encoded is in the surrogate pair range, we need to
556 * make sure there is another escape that we can use also.
558 uint32_t codePoint = readHex();
559 if (codePoint >= 0xd800 && codePoint <= 0xdbff) {
560 if (!in.consume("\\u")) {
561 in.error("expected another unicode escape for second half of "
564 uint16_t second = readHex();
565 if (second >= 0xdc00 && second <= 0xdfff) {
566 codePoint = 0x10000 + ((codePoint & 0x3ff) << 10) +
569 in.error("second character in surrogate pair is invalid");
571 } else if (codePoint >= 0xdc00 && codePoint <= 0xdfff) {
572 in.error("invalid unicode code point (in range [0xdc00,0xdfff])");
575 return codePointToUtf8(codePoint);
578 fbstring parseString(Input& in) {
584 auto range = in.skipWhile(
585 [] (char c) { return c != '\"' && c != '\\'; }
587 ret.append(range.begin(), range.end());
596 case '\"': ret.push_back('\"'); ++in; break;
597 case '\\': ret.push_back('\\'); ++in; break;
598 case '/': ret.push_back('/'); ++in; break;
599 case 'b': ret.push_back('\b'); ++in; break;
600 case 'f': ret.push_back('\f'); ++in; break;
601 case 'n': ret.push_back('\n'); ++in; break;
602 case 'r': ret.push_back('\r'); ++in; break;
603 case 't': ret.push_back('\t'); ++in; break;
604 case 'u': ++in; ret += decodeUnicodeEscape(in); break;
605 default: in.error(to<fbstring>("unknown escape ", *in,
606 " in string").c_str());
611 in.error("unterminated string");
615 * Apparently we're actually supposed to ban all control
616 * characters from strings. This seems unnecessarily
617 * restrictive, so we're only banning zero bytes. (Since the
618 * string is presumed to be UTF-8 encoded it's fine to just
621 in.error("null byte in string");
631 dynamic parseValue(Input& in) {
633 return *in == '[' ? parseArray(in) :
634 *in == '{' ? parseObject(in) :
635 *in == '\"' ? parseString(in) :
636 (*in == '-' || (*in >= '0' && *in <= '9')) ? parseNumber(in) :
637 in.consume("true") ? true :
638 in.consume("false") ? false :
639 in.consume("null") ? nullptr :
640 in.consume("Infinity") ?
641 (in.getOpts().parse_numbers_as_strings ? (dynamic)"Infinity" :
642 (dynamic)std::numeric_limits<double>::infinity()) :
644 (in.getOpts().parse_numbers_as_strings ? (dynamic)"NaN" :
645 (dynamic)std::numeric_limits<double>::quiet_NaN()) :
646 in.error("expected json value");
651 //////////////////////////////////////////////////////////////////////
653 fbstring serialize(dynamic const& dyn, serialization_opts const& opts) {
655 unsigned indentLevel = 0;
656 Printer p(ret, opts.pretty_formatting ? &indentLevel : nullptr, &opts);
661 // Escape a string so that it is legal to print it in JSON text.
662 void escapeString(StringPiece input,
664 const serialization_opts& opts) {
665 auto hexDigit = [] (int c) -> char {
666 return c < 10 ? c + '0' : c - 10 + 'a';
669 out.reserve(out.size() + input.size() + 2);
672 auto* p = reinterpret_cast<const unsigned char*>(input.begin());
673 auto* q = reinterpret_cast<const unsigned char*>(input.begin());
674 auto* e = reinterpret_cast<const unsigned char*>(input.end());
677 // Since non-ascii encoding inherently does utf8 validation
678 // we explicitly validate utf8 only if non-ascii encoding is disabled.
679 if ((opts.validate_utf8 || opts.skip_invalid_utf8)
680 && !opts.encode_non_ascii) {
681 // to achieve better spatial and temporal coherence
682 // we do utf8 validation progressively along with the
683 // string-escaping instead of two separate passes
685 // as the encoding progresses, q will stay at or ahead of p
688 // as p catches up with q, move q forward
690 // calling utf8_decode has the side effect of
691 // checking that utf8 encodings are valid
692 char32_t v = decodeUtf8(q, e, opts.skip_invalid_utf8);
693 if (opts.skip_invalid_utf8 && v == U'\ufffd') {
694 out.append(u8"\ufffd");
700 if (opts.encode_non_ascii && (*p & 0x80)) {
701 // note that this if condition captures utf8 chars
702 // with value > 127, so size > 1 byte
703 char32_t v = decodeUtf8(p, e, opts.skip_invalid_utf8);
705 out.push_back(hexDigit(v >> 12));
706 out.push_back(hexDigit((v >> 8) & 0x0f));
707 out.push_back(hexDigit((v >> 4) & 0x0f));
708 out.push_back(hexDigit(v & 0x0f));
709 } else if (*p == '\\' || *p == '\"') {
712 } else if (*p <= 0x1f) {
714 case '\b': out.append("\\b"); p++; break;
715 case '\f': out.append("\\f"); p++; break;
716 case '\n': out.append("\\n"); p++; break;
717 case '\r': out.append("\\r"); p++; break;
718 case '\t': out.append("\\t"); p++; break;
720 // note that this if condition captures non readable chars
721 // with value < 32, so size = 1 byte (e.g control chars).
723 out.push_back(hexDigit((*p & 0xf0) >> 4));
724 out.push_back(hexDigit(*p & 0xf));
735 fbstring stripComments(StringPiece jsonC) {
742 } state = State::None;
744 for (size_t i = 0; i < jsonC.size(); ++i) {
745 auto s = jsonC.subpiece(i);
748 if (s.startsWith("/*")) {
749 state = State::InlineComment;
752 } else if (s.startsWith("//")) {
753 state = State::LineComment;
756 } else if (s[0] == '\"') {
757 state = State::InString;
759 result.push_back(s[0]);
761 case State::InString:
763 if (UNLIKELY(s.size() == 1)) {
764 throw std::logic_error("Invalid JSONC: string is not terminated");
766 result.push_back(s[0]);
767 result.push_back(s[1]);
770 } else if (s[0] == '\"') {
773 result.push_back(s[0]);
775 case State::InlineComment:
776 if (s.startsWith("*/")) {
781 case State::LineComment:
783 // skip the line break. It doesn't matter.
788 throw std::logic_error("Unknown comment state");
796 //////////////////////////////////////////////////////////////////////
798 dynamic parseJson(StringPiece range) {
799 return parseJson(range, json::serialization_opts());
804 json::serialization_opts const& opts) {
806 json::Input in(range, &opts);
808 auto ret = parseValue(in);
810 if (in.size() && *in != '\0') {
811 in.error("parsing didn't consume all input");
816 fbstring toJson(dynamic const& dyn) {
817 return json::serialize(dyn, json::serialization_opts());
820 fbstring toPrettyJson(dynamic const& dyn) {
821 json::serialization_opts opts;
822 opts.pretty_formatting = true;
823 return json::serialize(dyn, opts);
826 //////////////////////////////////////////////////////////////////////
827 // dynamic::print_as_pseudo_json() is implemented here for header
828 // ordering reasons (most of the dynamic implementation is in
829 // dynamic-inl.h, which we don't want to include json.h).
831 void dynamic::print_as_pseudo_json(std::ostream& out) const {
832 json::serialization_opts opts;
833 opts.allow_non_string_keys = true;
834 opts.allow_nan_inf = true;
835 out << json::serialize(*this, opts);
838 void PrintTo(const dynamic& dyn, std::ostream* os) {
839 json::serialization_opts opts;
840 opts.allow_nan_inf = true;
841 opts.allow_non_string_keys = true;
842 opts.pretty_formatting = true;
843 opts.sort_keys = true;
844 *os << json::serialize(dyn, opts);
847 //////////////////////////////////////////////////////////////////////