X-Git-Url: http://demsky.eecs.uci.edu/git/?a=blobdiff_plain;f=folly%2Fjson.cpp;h=d4f0771d98a7d75c6138e0836462e3244ac100f4;hb=fd915b73606e09a5f46a1bca0a5d3643a1567014;hp=7711a20332c85d05978554bd7c11919e29d69833;hpb=ea72b037796a892aa00ef23902186d8f6b5d2ae9;p=folly.git diff --git a/folly/json.cpp b/folly/json.cpp index 7711a203..d4f0771d 100644 --- a/folly/json.cpp +++ b/folly/json.cpp @@ -1,5 +1,5 @@ /* - * Copyright 2013 Facebook, Inc. + * Copyright 2014 Facebook, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,14 +14,15 @@ * limitations under the License. */ -#include "folly/json.h" +#include #include #include #include -#include "folly/Range.h" -#include "folly/Unicode.h" -#include "folly/Conv.h" +#include +#include +#include +#include namespace folly { @@ -30,7 +31,10 @@ namespace folly { namespace json { namespace { -char32_t decodeUtf8(const unsigned char*& p, const unsigned char* const e) { +char32_t decodeUtf8( + const unsigned char*& p, + const unsigned char* const e, + bool skipOnError) { /* The following encodings are valid, except for the 5 and 6 byte * combinations: * 0xxxxxxx @@ -41,7 +45,10 @@ char32_t decodeUtf8(const unsigned char*& p, const unsigned char* const e) { * 1111110x 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx */ + auto skip = [&] { ++p; return U'\ufffd'; }; + if (p >= e) { + if (skipOnError) return skip(); throw std::runtime_error("folly::decodeUtf8 empty/invalid string"); } @@ -62,8 +69,8 @@ char32_t decodeUtf8(const unsigned char*& p, const unsigned char* const e) { uint32_t d = fst; if ((fst & 0xC0) != 0xC0) { - throw std::runtime_error( - to("folly::decodeUtf8 i=0 d=", d)); + if (skipOnError) return skip(); + throw std::runtime_error(to("folly::decodeUtf8 i=0 d=", d)); } fst <<= 1; @@ -72,6 +79,7 @@ char32_t decodeUtf8(const unsigned char*& p, const unsigned char* const e) { unsigned char tmp = p[i]; if ((tmp & 0xC0) != 0x80) { + if (skipOnError) return skip(); throw std::runtime_error( to("folly::decodeUtf8 i=", i, " tmp=", (uint32_t)tmp)); } @@ -84,6 +92,7 @@ char32_t decodeUtf8(const unsigned char*& p, const unsigned char* const e) { // overlong, could have been encoded with i bytes if ((d & ~bitMask[i - 1]) == 0) { + if (skipOnError) return skip(); throw std::runtime_error( to("folly::decodeUtf8 i=", i, " d=", d)); } @@ -91,6 +100,7 @@ char32_t decodeUtf8(const unsigned char*& p, const unsigned char* const e) { // check for surrogates only needed for 3 bytes if (i == 2) { if ((d >= 0xD800 && d <= 0xDFFF) || d > 0x10FFFF) { + if (skipOnError) return skip(); throw std::runtime_error( to("folly::decodeUtf8 i=", i, " d=", d)); } @@ -101,6 +111,7 @@ char32_t decodeUtf8(const unsigned char*& p, const unsigned char* const e) { } } + if (skipOnError) return skip(); throw std::runtime_error("folly::decodeUtf8 encoding length maxed out"); } @@ -116,7 +127,12 @@ struct Printer { void operator()(dynamic const& v) const { switch (v.type()) { case dynamic::DOUBLE: - toAppend(v.asDouble(), &out_); + if (!opts_.allow_nan_inf && + (std::isnan(v.asDouble()) || std::isinf(v.asDouble()))) { + throw std::runtime_error("folly::toJson: JSON object value was a " + "NaN or INF"); + } + toAppend(v.asDouble(), &out_, opts_.double_mode, opts_.double_num_digits); break; case dynamic::INT64: { auto intval = v.asInt(); @@ -149,7 +165,7 @@ struct Printer { } private: - void printKV(const std::pair& p) const { + void printKV(const std::pair& p) const { if (!opts_.allow_non_string_keys && !p.first.isString()) { throw std::runtime_error("folly::toJson: JSON object key was not a " "string"); @@ -159,6 +175,16 @@ private: (*this)(p.second); } + template + void printKVPairs(Iterator begin, Iterator end) const { + printKV(*begin); + for (++begin; begin != end; ++begin) { + out_ += ','; + newline(); + printKV(*begin); + } + } + void printObject(dynamic const& o) const { if (o.empty()) { out_ += "{}"; @@ -168,12 +194,13 @@ private: out_ += '{'; indent(); newline(); - auto it = o.items().begin(); - printKV(*it); - for (++it; it != o.items().end(); ++it) { - out_ += ','; - newline(); - printKV(*it); + if (opts_.sort_keys) { + std::vector> items( + o.items().begin(), o.items().end()); + std::sort(items.begin(), items.end()); + printKVPairs(items.begin(), items.end()); + } else { + printKVPairs(o.items().begin(), o.items().end()); } outdent(); newline(); @@ -190,7 +217,7 @@ private: indent(); newline(); (*this)(a[0]); - for (auto& val : makeRange(boost::next(a.begin()), a.end())) { + for (auto& val : range(boost::next(a.begin()), a.end())) { out_ += ','; newline(); (*this)(val); @@ -229,26 +256,6 @@ private: serialization_opts const& opts_; }; -////////////////////////////////////////////////////////////////////// - -struct ParseError : std::runtime_error { - explicit ParseError(int line) - : std::runtime_error(to("json parse error on line ", line)) - {} - - explicit ParseError(int line, std::string const& context, - std::string const& expected) - : std::runtime_error(to("json parse error on line ", line, - !context.empty() ? to(" near `", context, '\'') - : "", - ": ", expected)) - {} - - explicit ParseError(std::string const& what) - : std::runtime_error("json parse error: " + what) - {} -}; - // Wraps our input buffer with some helper functions. struct Input { explicit Input(StringPiece range, json::serialization_opts const* opts) @@ -287,21 +294,17 @@ struct Input { return skipWhile([] (char c) { return c >= '0' && c <= '9'; }); } - void skipWhitespace() { - // Spaces other than ' ' characters are less common but should be - // checked. This configuration where we loop on the ' ' - // separately from oddspaces was empirically fastest. - auto oddspace = [] (char c) { - return c == '\n' || c == '\t' || c == '\r'; - }; + StringPiece skipMinusAndDigits() { + bool firstChar = true; + return skipWhile([&firstChar] (char c) { + bool result = (c >= '0' && c <= '9') || (firstChar && c == '-'); + firstChar = false; + return result; + }); + } - loop: - for (; !range_.empty() && range_.front() == ' '; range_.pop_front()) { - } - if (!range_.empty() && oddspace(range_.front())) { - range_.pop_front(); - goto loop; - } + void skipWhitespace() { + range_ = folly::skipWhitespace(range_); storeCurrent(); } @@ -448,22 +451,19 @@ dynamic parseArray(Input& in) { dynamic parseNumber(Input& in) { bool const negative = (*in == '-'); if (negative) { - ++in; - if (in.consume("Infinity")) { + if (in.consume("-Infinity")) { return -std::numeric_limits::infinity(); } } - auto integral = in.skipDigits(); - if (integral.empty()) { + auto integral = in.skipMinusAndDigits(); + if (negative && integral.size() < 2) { in.error("expected digits after `-'"); } + auto const wasE = *in == 'e' || *in == 'E'; if (*in != '.' && !wasE) { auto val = to(integral); - if (negative) { - val = -val; - } in.skipWhitespace(); return val; } @@ -477,12 +477,9 @@ dynamic parseNumber(Input& in) { auto expPart = in.skipDigits(); end = expPart.end(); } - auto fullNum = makeRange(integral.begin(), end); + auto fullNum = range(integral.begin(), end); auto val = to(fullNum); - if (negative) { - val *= -1; - } return val; } @@ -631,7 +628,8 @@ void escapeString(StringPiece input, while (p < e) { // Since non-ascii encoding inherently does utf8 validation // we explicitly validate utf8 only if non-ascii encoding is disabled. - if (opts.validate_utf8 && !opts.encode_non_ascii) { + if ((opts.validate_utf8 || opts.skip_invalid_utf8) + && !opts.encode_non_ascii) { // to achieve better spatial and temporal coherence // we do utf8 validation progressively along with the // string-escaping instead of two separate passes @@ -643,13 +641,18 @@ void escapeString(StringPiece input, if (q == p) { // calling utf8_decode has the side effect of // checking that utf8 encodings are valid - decodeUtf8(q, e); + char32_t v = decodeUtf8(q, e, opts.skip_invalid_utf8); + if (opts.skip_invalid_utf8 && v == U'\ufffd') { + out.append("\ufffd"); + p = q; + continue; + } } } if (opts.encode_non_ascii && (*p & 0x80)) { // note that this if condition captures utf8 chars // with value > 127, so size > 1 byte - char32_t v = decodeUtf8(p, e); + char32_t v = decodeUtf8(p, e, opts.skip_invalid_utf8); out.append("\\u"); out.push_back(hexDigit(v >> 12)); out.push_back(hexDigit((v >> 8) & 0x0f)); @@ -681,6 +684,65 @@ void escapeString(StringPiece input, out.push_back('\"'); } +fbstring stripComments(StringPiece jsonC) { + fbstring result; + enum class State { + None, + InString, + InlineComment, + LineComment + } state = State::None; + + for (size_t i = 0; i < jsonC.size(); ++i) { + auto s = jsonC.subpiece(i); + switch (state) { + case State::None: + if (s.startsWith("/*")) { + state = State::InlineComment; + ++i; + continue; + } else if (s.startsWith("//")) { + state = State::LineComment; + ++i; + continue; + } else if (s[0] == '\"') { + state = State::InString; + } + result.push_back(s[0]); + break; + case State::InString: + if (s[0] == '\\') { + if (UNLIKELY(s.size() == 1)) { + throw std::logic_error("Invalid JSONC: string is not terminated"); + } + result.push_back(s[0]); + result.push_back(s[1]); + ++i; + continue; + } else if (s[0] == '\"') { + state = State::None; + } + result.push_back(s[0]); + break; + case State::InlineComment: + if (s.startsWith("*/")) { + state = State::None; + ++i; + } + break; + case State::LineComment: + if (s[0] == '\n') { + // skip the line break. It doesn't matter. + state = State::None; + } + break; + default: + throw std::logic_error("Unknown comment state"); + } + } + return result; +} + } ////////////////////////////////////////////////////////////////////// @@ -721,6 +783,7 @@ fbstring toPrettyJson(dynamic const& dyn) { void dynamic::print_as_pseudo_json(std::ostream& out) const { json::serialization_opts opts; opts.allow_non_string_keys = true; + opts.allow_nan_inf = true; out << json::serialize(*this, opts); }