2 * Copyright 2014 Facebook, Inc.
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
17 #include "folly/json.h"
18 #include <gtest/gtest.h>
19 #include <gflags/gflags.h>
23 #include <boost/next_prior.hpp>
24 #include "folly/Benchmark.h"
27 using folly::parseJson;
31 auto val = parseJson("\"I \u2665 UTF-8\"");
32 EXPECT_EQ("I \u2665 UTF-8", val.asString());
33 val = parseJson("\"I \\u2665 UTF-8\"");
34 EXPECT_EQ("I \u2665 UTF-8", val.asString());
35 val = parseJson("\"I \U0001D11E playing in G-clef\"");
36 EXPECT_EQ("I \U0001D11E playing in G-clef", val.asString());
38 val = parseJson("\"I \\uD834\\uDD1E playing in G-clef\"");
39 EXPECT_EQ("I \U0001D11E playing in G-clef", val.asString());
43 auto num = parseJson("12");
44 EXPECT_TRUE(num.isInt());
46 num = parseJson("12e5");
47 EXPECT_TRUE(num.isDouble());
49 auto numAs1 = num.asDouble();
50 EXPECT_EQ(numAs1, 12e5);
52 EXPECT_EQ(num, 1200000);
54 auto largeNumber = parseJson("4611686018427387904");
55 EXPECT_TRUE(largeNumber.isInt());
56 EXPECT_EQ(largeNumber, 4611686018427387904L);
58 auto negative = parseJson("-123");
59 EXPECT_EQ(negative, -123);
61 auto bfalse = parseJson("false");
62 auto btrue = parseJson("true");
63 EXPECT_EQ(bfalse, false);
64 EXPECT_EQ(btrue, true);
66 auto null = parseJson("null");
67 EXPECT_TRUE(null == nullptr);
69 auto doub1 = parseJson("12.0");
70 auto doub2 = parseJson("12e2");
71 EXPECT_EQ(doub1, 12.0);
72 EXPECT_EQ(doub2, 12e2);
73 EXPECT_EQ(std::numeric_limits<double>::infinity(),
74 parseJson("Infinity").asDouble());
75 EXPECT_EQ(-std::numeric_limits<double>::infinity(),
76 parseJson("-Infinity").asDouble());
77 EXPECT_TRUE(std::isnan(parseJson("NaN").asDouble()));
80 EXPECT_THROW(parseJson("infinity"), std::runtime_error);
81 EXPECT_THROW(parseJson("inf"), std::runtime_error);
82 EXPECT_THROW(parseJson("nan"), std::runtime_error);
84 auto array = parseJson(
85 "[12,false, false , null , [12e4,32, [], 12]]");
86 EXPECT_EQ(array.size(), 5);
87 if (array.size() == 5) {
88 EXPECT_EQ(boost::prior(array.end())->size(), 4);
91 EXPECT_THROW(parseJson("\n[12,\n\nnotvalidjson"),
94 EXPECT_THROW(parseJson("12e2e2"),
97 EXPECT_THROW(parseJson("{\"foo\":12,\"bar\":42} \"something\""),
100 dynamic value = dynamic::object
106 dynamic::object("a", "b")
116 // Print then parse and get the same thing, hopefully.
117 EXPECT_EQ(value, parseJson(toJson(value)));
120 // Test an object with non-string values.
121 dynamic something = parseJson(
122 "{\"old_value\":40,\"changed\":true,\"opened\":false}");
123 dynamic expected = dynamic::object
127 EXPECT_EQ(something, expected);
130 TEST(Json, ParseTrailingComma) {
131 folly::json::serialization_opts on, off;
132 on.allow_trailing_comma = true;
133 off.allow_trailing_comma = false;
135 dynamic arr { 1, 2 };
136 EXPECT_EQ(arr, parseJson("[1, 2]", on));
137 EXPECT_EQ(arr, parseJson("[1, 2,]", on));
138 EXPECT_EQ(arr, parseJson("[1, 2, ]", on));
139 EXPECT_EQ(arr, parseJson("[1, 2 , ]", on));
140 EXPECT_EQ(arr, parseJson("[1, 2 ,]", on));
141 EXPECT_THROW(parseJson("[1, 2,]", off), std::runtime_error);
143 dynamic obj = dynamic::object("a", 1);
144 EXPECT_EQ(obj, parseJson("{\"a\": 1}", on));
145 EXPECT_EQ(obj, parseJson("{\"a\": 1,}", on));
146 EXPECT_EQ(obj, parseJson("{\"a\": 1, }", on));
147 EXPECT_EQ(obj, parseJson("{\"a\": 1 , }", on));
148 EXPECT_EQ(obj, parseJson("{\"a\": 1 ,}", on));
149 EXPECT_THROW(parseJson("{\"a\":1,}", off), std::runtime_error);
152 TEST(Json, JavascriptSafe) {
153 auto badDouble = (1ll << 63ll) + 1;
154 dynamic badDyn = badDouble;
155 EXPECT_EQ(folly::toJson(badDouble), folly::to<folly::fbstring>(badDouble));
156 folly::json::serialization_opts opts;
157 opts.javascript_safe = true;
158 EXPECT_ANY_THROW(folly::json::serialize(badDouble, opts));
160 auto okDouble = 1ll << 63ll;
161 dynamic okDyn = okDouble;
162 EXPECT_EQ(folly::toJson(okDouble), folly::to<folly::fbstring>(okDouble));
165 TEST(Json, Produce) {
166 auto value = parseJson(R"( "f\"oo" )");
167 EXPECT_EQ(toJson(value), R"("f\"oo")");
168 value = parseJson("\"Control code: \001 \002 \x1f\"");
169 EXPECT_EQ(toJson(value), R"("Control code: \u0001 \u0002 \u001f")");
171 // We're not allowed to have non-string keys in json.
172 EXPECT_THROW(toJson(dynamic::object("abc", "xyz")(42.33, "asd")),
176 TEST(Json, JsonEscape) {
177 folly::json::serialization_opts opts;
179 folly::json::serialize("\b\f\n\r\x01\t\\\"/\v\a", opts),
180 R"("\b\f\n\r\u0001\t\\\"/\u000b\u0007")");
183 TEST(Json, JsonNonAsciiEncoding) {
184 folly::json::serialization_opts opts;
185 opts.encode_non_ascii = true;
188 EXPECT_EQ(folly::json::serialize("\x1f", opts), R"("\u001f")");
189 EXPECT_EQ(folly::json::serialize("\xc2\xa2", opts), R"("\u00a2")");
190 EXPECT_EQ(folly::json::serialize("\xe2\x82\xac", opts), R"("\u20ac")");
192 // multiple unicode encodings
194 folly::json::serialize("\x1f\xe2\x82\xac", opts),
195 R"("\u001f\u20ac")");
197 folly::json::serialize("\x1f\xc2\xa2\xe2\x82\xac", opts),
198 R"("\u001f\u00a2\u20ac")");
200 folly::json::serialize("\xc2\x80\xef\xbf\xbf", opts),
201 R"("\u0080\uffff")");
203 folly::json::serialize("\xe0\xa0\x80\xdf\xbf", opts),
204 R"("\u0800\u07ff")");
206 // first possible sequence of a certain length
207 EXPECT_EQ(folly::json::serialize("\xc2\x80", opts), R"("\u0080")");
208 EXPECT_EQ(folly::json::serialize("\xe0\xa0\x80", opts), R"("\u0800")");
210 // last possible sequence of a certain length
211 EXPECT_EQ(folly::json::serialize("\xdf\xbf", opts), R"("\u07ff")");
212 EXPECT_EQ(folly::json::serialize("\xef\xbf\xbf", opts), R"("\uffff")");
214 // other boundary conditions
215 EXPECT_EQ(folly::json::serialize("\xed\x9f\xbf", opts), R"("\ud7ff")");
216 EXPECT_EQ(folly::json::serialize("\xee\x80\x80", opts), R"("\ue000")");
217 EXPECT_EQ(folly::json::serialize("\xef\xbf\xbd", opts), R"("\ufffd")");
219 // incomplete sequences
220 EXPECT_ANY_THROW(folly::json::serialize("a\xed\x9f", opts));
221 EXPECT_ANY_THROW(folly::json::serialize("b\xee\x80", opts));
222 EXPECT_ANY_THROW(folly::json::serialize("c\xef\xbf", opts));
225 EXPECT_ANY_THROW(folly::json::serialize("\xfe", opts));
226 EXPECT_ANY_THROW(folly::json::serialize("\xff", opts));
228 // Sample overlong sequences
229 EXPECT_ANY_THROW(folly::json::serialize("\xc0\xaf", opts));
230 EXPECT_ANY_THROW(folly::json::serialize("\xe0\x80\xaf", opts));
232 // Maximum overlong sequences
233 EXPECT_ANY_THROW(folly::json::serialize("\xc1\xbf", opts));
234 EXPECT_ANY_THROW(folly::json::serialize("\x30\x9f\xbf", opts));
236 // illegal code positions
237 EXPECT_ANY_THROW(folly::json::serialize("\xed\xa0\x80", opts));
238 EXPECT_ANY_THROW(folly::json::serialize("\xed\xbf\xbf", opts));
240 // Overlong representation of NUL character
241 EXPECT_ANY_THROW(folly::json::serialize("\xc0\x80", opts));
242 EXPECT_ANY_THROW(folly::json::serialize("\xe0\x80\x80", opts));
244 // Longer than 3 byte encodings
245 EXPECT_ANY_THROW(folly::json::serialize("\xf4\x8f\xbf\xbf", opts));
246 EXPECT_ANY_THROW(folly::json::serialize("\xed\xaf\xbf\xed\xbf\xbf", opts));
249 TEST(Json, UTF8Retention) {
251 // test retention with valid utf8 strings
252 folly::fbstring input = "\u2665";
253 folly::fbstring jsonInput = folly::toJson(input);
254 folly::fbstring output = folly::parseJson(jsonInput).asString();
255 folly::fbstring jsonOutput = folly::toJson(output);
257 EXPECT_EQ(input, output);
258 EXPECT_EQ(jsonInput, jsonOutput);
260 // test retention with invalid utf8 - note that non-ascii chars are retained
261 // as is, and no unicode encoding is attempted so no exception is thrown.
263 folly::toJson("a\xe0\xa0\x80z\xc0\x80"),
264 "\"a\xe0\xa0\x80z\xc0\x80\""
268 TEST(Json, UTF8EncodeNonAsciiRetention) {
270 folly::json::serialization_opts opts;
271 opts.encode_non_ascii = true;
273 // test encode_non_ascii valid utf8 strings
274 folly::fbstring input = "\u2665";
275 folly::fbstring jsonInput = folly::json::serialize(input, opts);
276 folly::fbstring output = folly::parseJson(jsonInput).asString();
277 folly::fbstring jsonOutput = folly::json::serialize(output, opts);
279 EXPECT_EQ(input, output);
280 EXPECT_EQ(jsonInput, jsonOutput);
282 // test encode_non_ascii with invalid utf8 - note that an attempt to encode
283 // non-ascii to unicode will result is a utf8 validation and throw exceptions.
284 EXPECT_ANY_THROW(folly::json::serialize("a\xe0\xa0\x80z\xc0\x80", opts));
285 EXPECT_ANY_THROW(folly::json::serialize("a\xe0\xa0\x80z\xe0\x80\x80", opts));
288 TEST(Json, UTF8Validation) {
289 folly::json::serialization_opts opts;
290 opts.validate_utf8 = true;
292 // test validate_utf8 valid utf8 strings - note that we only validate the
293 // for utf8 but don't encode non-ascii to unicode so they are retained as is.
294 EXPECT_EQ(folly::json::serialize("a\xc2\x80z", opts), "\"a\xc2\x80z\"");
296 folly::json::serialize("a\xe0\xa0\x80z", opts),
297 "\"a\xe0\xa0\x80z\"");
299 folly::json::serialize("a\xe0\xa0\x80m\xc2\x80z", opts),
300 "\"a\xe0\xa0\x80m\xc2\x80z\"");
302 // test validate_utf8 with invalid utf8
303 EXPECT_ANY_THROW(folly::json::serialize("a\xe0\xa0\x80z\xc0\x80", opts));
304 EXPECT_ANY_THROW(folly::json::serialize("a\xe0\xa0\x80z\xe0\x80\x80", opts));
306 opts.skip_invalid_utf8 = true;
307 EXPECT_EQ(folly::json::serialize("a\xe0\xa0\x80z\xc0\x80", opts),
308 "\"a\xe0\xa0\x80z\ufffd\ufffd\"");
309 EXPECT_EQ(folly::json::serialize("a\xe0\xa0\x80z\xc0\x80\x80", opts),
310 "\"a\xe0\xa0\x80z\ufffd\ufffd\ufffd\"");
311 EXPECT_EQ(folly::json::serialize("z\xc0\x80z\xe0\xa0\x80", opts),
312 "\"z\ufffd\ufffdz\xe0\xa0\x80\"");
314 opts.encode_non_ascii = true;
315 EXPECT_EQ(folly::json::serialize("a\xe0\xa0\x80z\xc0\x80", opts),
316 "\"a\\u0800z\\ufffd\\ufffd\"");
317 EXPECT_EQ(folly::json::serialize("a\xe0\xa0\x80z\xc0\x80\x80", opts),
318 "\"a\\u0800z\\ufffd\\ufffd\\ufffd\"");
319 EXPECT_EQ(folly::json::serialize("z\xc0\x80z\xe0\xa0\x80", opts),
320 "\"z\\ufffd\\ufffdz\\u0800\"");
325 TEST(Json, ParseNonStringKeys) {
327 EXPECT_EQ("a", parseJson("{\"a\":[]}").items().begin()->first.asString());
329 // check that we don't allow non-string keys as this violates the
330 // strict JSON spec (though it is emitted by the output of
331 // folly::dynamic with operator <<).
332 EXPECT_THROW(parseJson("{1:[]}"), std::runtime_error);
334 // check that we can parse colloquial JSON if the option is set
335 folly::json::serialization_opts opts;
336 opts.allow_non_string_keys = true;
338 auto val = parseJson("{1:[]}", opts);
339 EXPECT_EQ(1, val.items().begin()->first.asInt());
342 // test we can still read in strings
343 auto sval = parseJson("{\"a\":[]}", opts);
344 EXPECT_EQ("a", sval.items().begin()->first.asString());
346 // test we can read in doubles
347 auto dval = parseJson("{1.5:[]}", opts);
348 EXPECT_EQ(1.5, dval.items().begin()->first.asDouble());
351 TEST(Json, SortKeys) {
352 folly::json::serialization_opts opts_on, opts_off;
353 opts_on.sort_keys = true;
354 opts_off.sort_keys = false;
356 dynamic value = dynamic::object
362 dynamic::object("a", "b")
372 std::string sorted_keys =
373 R"({"a":[{"a":"b","c":"d"},12.5,"Yo Dawg",["heh"],null],)"
374 R"("another":32.2,"foo":"bar","junk":12})";
376 EXPECT_EQ(value, parseJson(folly::json::serialize(value, opts_on)));
377 EXPECT_EQ(value, parseJson(folly::json::serialize(value, opts_off)));
379 EXPECT_EQ(sorted_keys, folly::json::serialize(value, opts_on));
382 BENCHMARK(jsonSerialize, iters) {
383 folly::json::serialization_opts opts;
384 for (int i = 0; i < iters; ++i) {
385 folly::json::serialize(
386 "qwerty \xc2\x80 \xef\xbf\xbf poiuy"
387 "qwerty \xc2\x80 \xef\xbf\xbf poiuy"
388 "qwerty \xc2\x80 \xef\xbf\xbf poiuy"
389 "qwerty \xc2\x80 \xef\xbf\xbf poiuy"
390 "qwerty \xc2\x80 \xef\xbf\xbf poiuy"
391 "qwerty \xc2\x80 \xef\xbf\xbf poiuy"
392 "qwerty \xc2\x80 \xef\xbf\xbf poiuy"
393 "qwerty \xc2\x80 \xef\xbf\xbf poiuy"
394 "qwerty \xc2\x80 \xef\xbf\xbf poiuy"
395 "qwerty \xc2\x80 \xef\xbf\xbf poiuy",
400 BENCHMARK(jsonSerializeWithNonAsciiEncoding, iters) {
401 folly::json::serialization_opts opts;
402 opts.encode_non_ascii = true;
404 for (int i = 0; i < iters; ++i) {
405 folly::json::serialize(
406 "qwerty \xc2\x80 \xef\xbf\xbf poiuy"
407 "qwerty \xc2\x80 \xef\xbf\xbf poiuy"
408 "qwerty \xc2\x80 \xef\xbf\xbf poiuy"
409 "qwerty \xc2\x80 \xef\xbf\xbf poiuy"
410 "qwerty \xc2\x80 \xef\xbf\xbf poiuy"
411 "qwerty \xc2\x80 \xef\xbf\xbf poiuy"
412 "qwerty \xc2\x80 \xef\xbf\xbf poiuy"
413 "qwerty \xc2\x80 \xef\xbf\xbf poiuy"
414 "qwerty \xc2\x80 \xef\xbf\xbf poiuy"
415 "qwerty \xc2\x80 \xef\xbf\xbf poiuy",
420 BENCHMARK(jsonSerializeWithUtf8Validation, iters) {
421 folly::json::serialization_opts opts;
422 opts.validate_utf8 = true;
424 for (int i = 0; i < iters; ++i) {
425 folly::json::serialize(
426 "qwerty \xc2\x80 \xef\xbf\xbf poiuy"
427 "qwerty \xc2\x80 \xef\xbf\xbf poiuy"
428 "qwerty \xc2\x80 \xef\xbf\xbf poiuy"
429 "qwerty \xc2\x80 \xef\xbf\xbf poiuy"
430 "qwerty \xc2\x80 \xef\xbf\xbf poiuy"
431 "qwerty \xc2\x80 \xef\xbf\xbf poiuy"
432 "qwerty \xc2\x80 \xef\xbf\xbf poiuy"
433 "qwerty \xc2\x80 \xef\xbf\xbf poiuy"
434 "qwerty \xc2\x80 \xef\xbf\xbf poiuy"
435 "qwerty \xc2\x80 \xef\xbf\xbf poiuy",
440 BENCHMARK(parseSmallStringWithUtf, iters) {
441 for (int i = 0; i < iters << 4; ++i) {
442 parseJson("\"I \\u2665 UTF-8 thjasdhkjh blah blah blah\"");
446 BENCHMARK(parseNormalString, iters) {
447 for (int i = 0; i < iters << 4; ++i) {
448 parseJson("\"akjhfk jhkjlakjhfk jhkjlakjhfk jhkjl akjhfk\"");
452 BENCHMARK(parseBigString, iters) {
453 for (int i = 0; i < iters; ++i) {
455 "akjhfk jhkjlakjhfk jhkjlakjhfk jhkjl akjhfk"
456 "akjhfk jhkjlakjhfk jhkjlakjhfk jhkjl akjhfk"
457 "akjhfk jhkjlakjhfk jhkjlakjhfk jhkjl akjhfk"
458 "akjhfk jhkjlakjhfk jhkjlakjhfk jhkjl akjhfk"
459 "akjhfk jhkjlakjhfk jhkjlakjhfk jhkjl akjhfk"
460 "akjhfk jhkjlakjhfk jhkjlakjhfk jhkjl akjhfk"
461 "akjhfk jhkjlakjhfk jhkjlakjhfk jhkjl akjhfk"
462 "akjhfk jhkjlakjhfk jhkjlakjhfk jhkjl akjhfk"
463 "akjhfk jhkjlakjhfk jhkjlakjhfk jhkjl akjhfk"
464 "akjhfk jhkjlakjhfk jhkjlakjhfk jhkjl akjhfk"
465 "akjhfk jhkjlakjhfk jhkjlakjhfk jhkjl akjhfk"
470 BENCHMARK(toJson, iters) {
471 dynamic something = parseJson(
472 "{\"old_value\":40,\"changed\":true,\"opened\":false,\"foo\":[1,2,3,4,5,6]}"
475 for (int i = 0; i < iters; i++) {
480 int main(int argc, char** argv) {
481 testing::InitGoogleTest(&argc, argv);
482 google::ParseCommandLineFlags(&argc, &argv, true);
483 if (FLAGS_benchmark) {
484 folly::runBenchmarks();
486 return RUN_ALL_TESTS();