Option to fallback to double when int precision is not enough.

Summary: Some libraries like Haskell's `Data.Aeson` can produce arbitrarily
big numbers with arbitrary precision.
The json standard doesn't specify the ranges for numeric types.
For interoperability, we should allow the user to parse the numbers with
some loss of precision.

Reviewed By: luciang

Differential Revision: D2565140

fb-gh-sync-id: b1a9a46e298bf13cc89d7e79ce28705e9e251a7f

diff --git a/folly/json.cpp b/folly/json.cpp
index fe06001cfee9c75ac7a7c199d49b06840c9568b6..4c856598e123bc61a1a8ea8760fdf69ddbf14fdc 100644 (file)
--- a/folly/json.cpp
+++ b/folly/json.cpp
@@ -482,10 +482,23 @@ dynamic parseNumber(Input& in) {
   }
 
   auto const wasE = *in == 'e' || *in == 'E';
+
+  constexpr const char* maxInt = "9223372036854775807";
+  constexpr const char* minInt = "9223372036854775808";
+  constexpr auto maxIntLen = __builtin_strlen(maxInt);
+
   if (*in != '.' && !wasE) {
-    auto val = to<int64_t>(integral);
-    in.skipWhitespace();
-    return val;
+    if (LIKELY(!in.getOpts().double_fallback || integral.size() < maxIntLen) ||
+         (integral.size() == maxIntLen &&
+           (integral <= maxInt || (integral == minInt && negative)))) {
+      auto val = to<int64_t>(integral);
+      in.skipWhitespace();
+      return val;
+    } else {
+      auto val = to<double>(integral);
+      in.skipWhitespace();
+      return val;
+    }
   }
 
   auto end = !wasE ? (++in, in.skipDigits().end()) : in.begin();

diff --git a/folly/json.h b/folly/json.h
index c0a6eeb2302797072058ef1d4534e3fa5c4fc4f9..8352abbf969371fd1ba167cfc77310f92c6d35b2 100644 (file)
--- a/folly/json.h
+++ b/folly/json.h
@@ -64,6 +64,7 @@ namespace json {
       , allow_nan_inf(false)
       , double_mode(double_conversion::DoubleToStringConverter::SHORTEST)
       , double_num_digits(0) // ignored when mode is SHORTEST
+      , double_fallback(false)
     {}
 
     // If true, keys in an object can be non-strings.  (In strict
@@ -104,6 +105,10 @@ namespace json {
     // toAppend implementation for floating point for more info
     double_conversion::DoubleToStringConverter::DtoaMode double_mode;
     unsigned int double_num_digits;
+
+    // Fallback to double when a value that looks like integer is too big to
+    // fit in an int64_t. Can result in loss a of precision.
+    bool double_fallback;
   };
 
   /*

diff --git a/folly/test/JsonTest.cpp b/folly/test/JsonTest.cpp
index 8b5964f6ab5cd9903a0b4a7da6f168eb72c1f4fb..02976cf3887203beff3c58e475b374c1a368b226 100644 (file)
--- a/folly/test/JsonTest.cpp
+++ b/folly/test/JsonTest.cpp
@@ -361,6 +361,46 @@ TEST(Json, ParseNonStringKeys) {
   EXPECT_EQ(1.5, dval.items().begin()->first.asDouble());
 }
 
+TEST(Json, ParseDoubleFallback) {
+  // default behavior
+  EXPECT_THROW(parseJson("{\"a\":847605071342477600000000000000}"),
+      std::range_error);
+  EXPECT_THROW(parseJson("{\"a\":-9223372036854775809}"),
+      std::range_error);
+  EXPECT_THROW(parseJson("{\"a\":9223372036854775808}"),
+      std::range_error);
+  EXPECT_EQ(std::numeric_limits<int64_t>::min(),
+      parseJson("{\"a\":-9223372036854775808}").items().begin()
+        ->second.asInt());
+  EXPECT_EQ(std::numeric_limits<int64_t>::max(),
+      parseJson("{\"a\":9223372036854775807}").items().begin()->second.asInt());
+  // with double_fallback
+  folly::json::serialization_opts opts;
+  opts.double_fallback = true;
+  EXPECT_EQ(847605071342477600000000000000.0,
+      parseJson("{\"a\":847605071342477600000000000000}",
+        opts).items().begin()->second.asDouble());
+  EXPECT_EQ(847605071342477600000000000000.0,
+      parseJson("{\"a\": 847605071342477600000000000000}",
+        opts).items().begin()->second.asDouble());
+  EXPECT_EQ(847605071342477600000000000000.0,
+      parseJson("{\"a\":847605071342477600000000000000 }",
+        opts).items().begin()->second.asDouble());
+  EXPECT_EQ(847605071342477600000000000000.0,
+      parseJson("{\"a\": 847605071342477600000000000000 }",
+        opts).items().begin()->second.asDouble());
+  EXPECT_EQ(std::numeric_limits<int64_t>::min(),
+      parseJson("{\"a\":-9223372036854775808}",
+        opts).items().begin()->second.asInt());
+  EXPECT_EQ(std::numeric_limits<int64_t>::max(),
+      parseJson("{\"a\":9223372036854775807}",
+        opts).items().begin()->second.asInt());
+  // show that some precision gets lost
+  EXPECT_EQ(847605071342477612345678900000.0,
+      parseJson("{\"a\":847605071342477612345678912345}",
+        opts).items().begin()->second.asDouble());
+}
+
 TEST(Json, SortKeys) {
   folly::json::serialization_opts opts_on, opts_off;
   opts_on.sort_keys = true;