experimental/EliasFanoCoding.h \
experimental/EventCount.h \
experimental/Instructions.h \
+ experimental/bser/Bser.h \
experimental/fibers/AddTasks.h \
experimental/fibers/AddTasks-inl.h \
experimental/fibers/Baton.h \
TimeoutQueue.cpp \
Uri.cpp \
Version.cpp \
+ experimental/bser/Dump.cpp \
+ experimental/bser/Load.cpp \
experimental/fibers/Baton.cpp \
experimental/fibers/Fiber.cpp \
experimental/fibers/FiberManager.cpp \
--- /dev/null
+/*
+ * Copyright 2016 Facebook, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#pragma once
+#include <folly/Optional.h>
+#include <folly/dynamic.h>
+#include <folly/io/IOBuf.h>
+#include <folly/io/IOBufQueue.h>
+#include <unordered_map>
+
+/* This is an implementation of the BSER binary serialization scheme.
+ * BSER was created as a binary, local-system-only representation of
+ * JSON values. It is more space efficient in its output text than JSON,
+ * and cheaper to decode.
+ * It has no requirement that string values be UTF-8.
+ * BSER was created for use with Watchman.
+ * https://facebook.github.io/watchman/docs/bser.html
+ */
+
+namespace folly {
+namespace bser {
+
+class BserDecodeError : public std::runtime_error {
+ public:
+ using std::runtime_error::runtime_error;
+};
+
+enum class BserType : int8_t {
+ Array = 0,
+ Object,
+ String,
+ Int8,
+ Int16,
+ Int32,
+ Int64,
+ Real,
+ True,
+ False,
+ Null,
+ Template,
+ Skip,
+};
+extern const uint8_t kMagic[2];
+
+struct serialization_opts {
+ serialization_opts();
+
+ // Whether to sort keys of object values before serializing them.
+ // Note that this is potentially slow and that it does not apply
+ // to templated arrays defined via defineTemplate; its keys are always
+ // emitted in the order defined by the template.
+ bool sort_keys;
+
+ // incremental growth size for the underlying Appender when allocating
+ // storage for the encoded output
+ size_t growth_increment;
+
+ // BSER allows generating a more space efficient representation of a list of
+ // object values. These are stored as an "object template" listing the keys
+ // of the objects ahead of the objects themselves. The objects are then
+ // serialized without repeating the key string for each element.
+ //
+ // You may use the templates field to associate a template with an
+ // array. You should construct this map after all mutations have been
+ // performed on the dynamic instance that you intend to serialize as bser,
+ // as it captures the address of the dynamic to match at encoding time.
+ // https://facebook.github.io/watchman/docs/bser.html#array-of-templated-objects
+ using TemplateMap = std::unordered_map<const folly::dynamic*, folly::dynamic>;
+ folly::Optional<TemplateMap> templates;
+};
+
+// parse a BSER value from a variety of sources.
+// The complete BSER data must be present to succeed.
+folly::dynamic parseBser(folly::StringPiece);
+folly::dynamic parseBser(folly::ByteRange);
+folly::dynamic parseBser(const folly::IOBuf*);
+
+// When reading incrementally, it is useful to know how much data to
+// read to fully decode a BSER pdu.
+// Throws std::out_of_range if more data needs to be read to decode
+// the header, or throws a runtime_error if the header is invalid
+size_t decodePduLength(const folly::IOBuf*);
+
+folly::fbstring toBser(folly::dynamic const&, const serialization_opts&);
+std::unique_ptr<folly::IOBuf> toBserIOBuf(folly::dynamic const&,
+ const serialization_opts&);
+}
+}
+
+/* vim:ts=2:sw=2:et:
+ */
--- /dev/null
+/*
+ * Copyright 2016 Facebook, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "Bser.h"
+#include <folly/io/Cursor.h>
+
+using namespace folly;
+using folly::io::QueueAppender;
+using folly::bser::serialization_opts;
+
+namespace folly {
+namespace bser {
+
+const uint8_t kMagic[2] = {0, 1};
+
+static void bserEncode(dynamic const& dyn,
+ QueueAppender& appender,
+ const serialization_opts& opts);
+
+serialization_opts::serialization_opts()
+ : sort_keys(false), growth_increment(8192) {}
+
+static const dynamic* getTemplate(const serialization_opts& opts,
+ dynamic const& dynArray) {
+ if (!opts.templates.hasValue()) {
+ return nullptr;
+ }
+ const auto& templates = opts.templates.value();
+ const auto it = templates.find(&dynArray);
+ if (it == templates.end()) {
+ return nullptr;
+ }
+ return &it->second;
+}
+
+static void bserEncodeInt(int64_t ival, QueueAppender& appender) {
+ /* Return the smallest size int that can store the value */
+ auto size =
+ ((ival == ((int8_t)ival)) ? 1 : (ival == ((int16_t)ival))
+ ? 2
+ : (ival == ((int32_t)ival)) ? 4 : 8);
+
+ switch (size) {
+ case 1:
+ appender.write((int8_t)BserType::Int8);
+ appender.write(int8_t(ival));
+ return;
+ case 2:
+ appender.write((int8_t)BserType::Int16);
+ appender.write(int16_t(ival));
+ return;
+ case 4:
+ appender.write((int8_t)BserType::Int32);
+ appender.write(int32_t(ival));
+ return;
+ case 8:
+ appender.write((int8_t)BserType::Int64);
+ appender.write(ival);
+ return;
+ default:
+ throw std::runtime_error("impossible integer size");
+ }
+}
+
+static void bserEncodeString(folly::StringPiece str, QueueAppender& appender) {
+ appender.write((int8_t)BserType::String);
+ bserEncodeInt(str.size(), appender);
+ appender.push((uint8_t*)str.data(), str.size());
+}
+
+static void bserEncodeArraySimple(dynamic const& dyn,
+ QueueAppender& appender,
+ const serialization_opts& opts) {
+ appender.write((int8_t)BserType::Array);
+ bserEncodeInt(dyn.size(), appender);
+ for (const auto& ele : dyn) {
+ bserEncode(ele, appender, opts);
+ }
+}
+
+static void bserEncodeArray(dynamic const& dyn,
+ QueueAppender& appender,
+ const serialization_opts& opts) {
+
+ auto templ = getTemplate(opts, dyn);
+ if (UNLIKELY(templ != nullptr)) {
+ appender.write((int8_t)BserType::Template);
+
+ // Emit the list of property names
+ bserEncodeArraySimple(*templ, appender, opts);
+
+ // The number of objects in the array
+ bserEncodeInt(dyn.size(), appender);
+
+ // For each object in the array
+ for (const auto& ele : dyn) {
+ // For each key in the template
+ for (const auto& name : *templ) {
+ if (auto found = ele.get_ptr(name)) {
+ if (found->isNull()) {
+ // Prefer to Skip rather than encode a null value for
+ // compatibility with the other bser implementations
+ appender.write((int8_t)BserType::Skip);
+ } else {
+ bserEncode(*found, appender, opts);
+ }
+ } else {
+ appender.write((int8_t)BserType::Skip);
+ }
+ }
+ }
+ return;
+ }
+
+ bserEncodeArraySimple(dyn, appender, opts);
+}
+
+static void bserEncodeObject(dynamic const& dyn,
+ QueueAppender& appender,
+ const serialization_opts& opts) {
+ appender.write((int8_t)BserType::Object);
+ bserEncodeInt(dyn.size(), appender);
+
+ if (opts.sort_keys) {
+ std::vector<std::pair<dynamic, dynamic>> sorted(dyn.items().begin(),
+ dyn.items().end());
+ std::sort(sorted.begin(), sorted.end());
+ for (const auto& item : sorted) {
+ bserEncode(item.first, appender, opts);
+ bserEncode(item.second, appender, opts);
+ }
+ } else {
+ for (const auto& item : dyn.items()) {
+ bserEncode(item.first, appender, opts);
+ bserEncode(item.second, appender, opts);
+ }
+ }
+}
+
+static void bserEncode(dynamic const& dyn,
+ QueueAppender& appender,
+ const serialization_opts& opts) {
+ switch (dyn.type()) {
+ case dynamic::Type::NULLT:
+ appender.write((int8_t)BserType::Null);
+ return;
+ case dynamic::Type::BOOL:
+ appender.write(
+ (int8_t)(dyn.getBool() ? BserType::True : BserType::False));
+ return;
+ case dynamic::Type::DOUBLE: {
+ double dval = dyn.getDouble();
+ appender.write((int8_t)BserType::Real);
+ appender.write(dval);
+ return;
+ }
+ case dynamic::Type::INT64:
+ bserEncodeInt(dyn.getInt(), appender);
+ return;
+ case dynamic::Type::OBJECT:
+ bserEncodeObject(dyn, appender, opts);
+ return;
+ case dynamic::Type::ARRAY:
+ bserEncodeArray(dyn, appender, opts);
+ return;
+ case dynamic::Type::STRING:
+ bserEncodeString(dyn.getString(), appender);
+ return;
+ }
+}
+
+std::unique_ptr<folly::IOBuf> toBserIOBuf(folly::dynamic const& dyn,
+ const serialization_opts& opts) {
+ IOBufQueue q(IOBufQueue::cacheChainLength());
+ uint8_t hdrbuf[sizeof(kMagic) + 1 + sizeof(int64_t)];
+
+ // Reserve some headroom for the overall PDU size; we'll fill this in
+ // after we've serialized the data and know the length
+ auto firstbuf = IOBuf::create(opts.growth_increment);
+ firstbuf->advance(sizeof(hdrbuf));
+ q.append(std::move(firstbuf));
+
+ // encode the value
+ QueueAppender appender(&q, opts.growth_increment);
+ bserEncode(dyn, appender, opts);
+
+ // compute the length
+ auto len = q.chainLength();
+ if (len > std::numeric_limits<int64_t>::max()) {
+ throw std::range_error(folly::to<std::string>(
+ "serialized data size ", len, " is too large to represent as BSER"));
+ }
+
+ // This is a bit verbose, but it computes a header that is appropriate
+ // to the size of the serialized data
+
+ memcpy(hdrbuf, kMagic, sizeof(kMagic));
+ size_t hdrlen = sizeof(kMagic) + 1;
+ auto magicptr = hdrbuf + sizeof(kMagic);
+ auto lenptr = hdrbuf + hdrlen;
+
+ if (len > std::numeric_limits<int32_t>::max()) {
+ *magicptr = (int8_t)BserType::Int64;
+ *(int64_t*)lenptr = (int64_t)len;
+ hdrlen += sizeof(int64_t);
+ } else if (len > std::numeric_limits<int16_t>::max()) {
+ *magicptr = (int8_t)BserType::Int32;
+ *(int32_t*)lenptr = (int32_t)len;
+ hdrlen += sizeof(int32_t);
+ } else if (len > std::numeric_limits<int8_t>::max()) {
+ *magicptr = (int8_t)BserType::Int16;
+ *(int16_t*)lenptr = (int16_t)len;
+ hdrlen += sizeof(int16_t);
+ } else {
+ *magicptr = (int8_t)BserType::Int8;
+ *(int8_t*)lenptr = (int8_t)len;
+ hdrlen += sizeof(int8_t);
+ }
+
+ // and place the data in the headroom
+ q.prepend(hdrbuf, hdrlen);
+
+ return q.move();
+}
+
+fbstring toBser(dynamic const& dyn, const serialization_opts& opts) {
+ auto buf = toBserIOBuf(dyn, opts);
+ return buf->moveToFbString();
+}
+}
+}
+
+/* vim:ts=2:sw=2:et:
+ */
--- /dev/null
+/*
+ * Copyright 2016 Facebook, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "Bser.h"
+#include <folly/io/Cursor.h>
+#include <folly/String.h>
+
+using namespace folly;
+using folly::io::Cursor;
+
+namespace folly {
+namespace bser {
+static dynamic parseBser(Cursor& curs);
+
+template <typename... ARGS>
+static FOLLY_NORETURN void throwDecodeError(Cursor& curs, ARGS&&... args) {
+ throw BserDecodeError(folly::to<std::string>(std::forward<ARGS>(args)...,
+ " with ",
+ curs.length(),
+ " bytes remaining in cursor"));
+}
+
+static int64_t decodeInt(Cursor& curs) {
+ auto enc = (BserType)curs.read<int8_t>();
+ switch (enc) {
+ case BserType::Int8:
+ return curs.read<int8_t>();
+ case BserType::Int16:
+ return curs.read<int16_t>();
+ case BserType::Int32:
+ return curs.read<int32_t>();
+ case BserType::Int64:
+ return curs.read<int64_t>();
+ default:
+ throwDecodeError(
+ curs, "invalid integer encoding detected (", (int8_t)enc, ")");
+ }
+}
+
+static fbstring decodeString(Cursor& curs) {
+ auto len = decodeInt(curs);
+ folly::fbstring str;
+
+ if (len < 0) {
+ throw std::range_error("string length must not be negative");
+ }
+ str.reserve(len);
+
+ size_t available = curs.length();
+ while (available < (size_t)len) {
+ if (available == 0) {
+ // Saw this case when we decodeHeader was returning the incorrect length
+ // and we were splitting off too few bytes from the IOBufQueue
+ throwDecodeError(curs,
+ "no data available while decoding a string, header was "
+ "not decoded properly");
+ }
+ str.append(reinterpret_cast<const char*>(curs.data()), available);
+ curs.skipAtMost(available);
+ len -= available;
+ available = curs.length();
+ }
+
+ str.append(reinterpret_cast<const char*>(curs.data()), len);
+ curs.skipAtMost(len);
+ return str;
+}
+
+static dynamic decodeArray(Cursor& curs) {
+ dynamic arr{};
+ auto size = decodeInt(curs);
+ while (size-- > 0) {
+ arr.push_back(parseBser(curs));
+ }
+ return arr;
+}
+
+static dynamic decodeObject(Cursor& curs) {
+ dynamic obj = dynamic::object;
+ auto size = decodeInt(curs);
+ while (size-- > 0) {
+ if ((BserType)curs.read<int8_t>() != BserType::String) {
+ throwDecodeError(curs, "expected String");
+ }
+ auto key = decodeString(curs);
+ obj[key] = parseBser(curs);
+ }
+ return obj;
+}
+
+static dynamic decodeTemplate(Cursor& curs) {
+ std::vector<dynamic> arr;
+
+ // List of property names
+ if ((BserType)curs.read<int8_t>() != BserType::Array) {
+ throw std::runtime_error("Expected array encoding for property names");
+ }
+ auto names = decodeArray(curs);
+
+ auto size = decodeInt(curs);
+ arr.reserve(size);
+
+ while (size-- > 0) {
+ dynamic obj = dynamic::object;
+
+ for (auto& name : names) {
+ auto pair = curs.peek();
+ if ((BserType)pair.first[0] == BserType::Skip) {
+ obj[name.getString()] = nullptr;
+ curs.skipAtMost(1);
+ continue;
+ }
+
+ obj[name.getString()] = parseBser(curs);
+ }
+
+ arr.emplace_back(std::move(obj));
+ }
+
+ return dynamic(std::move(arr));
+}
+
+static dynamic parseBser(Cursor& curs) {
+ switch ((BserType)curs.read<int8_t>()) {
+ case BserType::Int8:
+ return curs.read<int8_t>();
+ case BserType::Int16:
+ return curs.read<int16_t>();
+ case BserType::Int32:
+ return curs.read<int32_t>();
+ case BserType::Int64:
+ return curs.read<int64_t>();
+ case BserType::Real: {
+ double dval;
+ curs.pull((void*)&dval, sizeof(dval));
+ return dval;
+ }
+ case BserType::Null:
+ return nullptr;
+ case BserType::True:
+ return (bool)true;
+ case BserType::False:
+ return (bool)false;
+ case BserType::String:
+ return decodeString(curs);
+ case BserType::Array:
+ return decodeArray(curs);
+ case BserType::Object:
+ return decodeObject(curs);
+ case BserType::Template:
+ return decodeTemplate(curs);
+ case BserType::Skip:
+ throw std::runtime_error(
+ "Skip not valid at this location in the bser stream");
+ default:
+ throw std::runtime_error("invalid bser encoding");
+ }
+}
+
+static size_t decodeHeader(Cursor& curs) {
+ char header[sizeof(kMagic)];
+ curs.pull(header, sizeof(header));
+ if (memcmp(header, kMagic, sizeof(kMagic))) {
+ throw std::runtime_error("invalid BSER magic header");
+ }
+
+ auto enc = (BserType)curs.peek().first[0];
+ size_t int_size;
+ switch (enc) {
+ case BserType::Int8:
+ int_size = 1;
+ break;
+ case BserType::Int16:
+ int_size = 2;
+ break;
+ case BserType::Int32:
+ int_size = 4;
+ break;
+ case BserType::Int64:
+ int_size = 8;
+ break;
+ default:
+ int_size = 0;
+ }
+
+ return int_size + 3 /* magic + int type */ + decodeInt(curs);
+}
+
+size_t decodePduLength(const folly::IOBuf* buf) {
+ Cursor curs(buf);
+ return decodeHeader(curs);
+}
+
+folly::dynamic parseBser(const IOBuf* buf) {
+ Cursor curs(buf);
+
+ decodeHeader(curs);
+ return parseBser(curs);
+}
+
+folly::dynamic parseBser(ByteRange str) {
+ auto buf = IOBuf::wrapBuffer(str.data(), str.size());
+ return parseBser(&*buf);
+}
+
+folly::dynamic parseBser(StringPiece str) {
+ return parseBser(ByteRange((uint8_t*)str.data(), str.size()));
+}
+}
+}
+
+/* vim:ts=2:sw=2:et:
+ */
--- /dev/null
+/*
+ * Copyright 2016 Facebook, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include <folly/experimental/bser/Bser.h>
+#include <folly/String.h>
+#include <gtest/gtest.h>
+
+using folly::dynamic;
+
+static const dynamic roundtrips[] = {
+ 1,
+ std::numeric_limits<int8_t>::max(),
+ std::numeric_limits<int16_t>::max(),
+ std::numeric_limits<int32_t>::max(),
+ std::numeric_limits<int64_t>::max(),
+ std::numeric_limits<int8_t>::min(),
+ std::numeric_limits<int16_t>::min(),
+ std::numeric_limits<int32_t>::min(),
+ std::numeric_limits<int64_t>::min(),
+ bool(true),
+ bool(false),
+ nullptr,
+ 1.5,
+ "hello",
+ {1, 2, 3},
+ dynamic::object("key", "value")("otherkey", "otherval"),
+};
+
+// Here's a blob from the watchman test suite
+const uint8_t template_blob[] =
+ "\x00\x01\x03\x28"
+ "\x0b\x00\x03\x02\x02\x03\x04\x6e\x61\x6d\x65\x02"
+ "\x03\x03\x61\x67\x65\x03\x03\x02\x03\x04\x66\x72"
+ "\x65\x64\x03\x14\x02\x03\x04\x70\x65\x74\x65\x03"
+ "\x1e\x0c\x03\x19";
+
+// and here's what it represents
+static const dynamic template_dynamic = {
+ dynamic::object("name", "fred")("age", 20),
+ dynamic::object("name", "pete")("age", 30),
+ dynamic::object("name", nullptr)("age", 25),
+};
+
+TEST(Bser, RoundTrip) {
+ dynamic decoded(nullptr);
+ folly::fbstring str;
+
+ for (const auto& dyn : roundtrips) {
+ try {
+ str = folly::bser::toBser(dyn, folly::bser::serialization_opts());
+ decoded = folly::bser::parseBser(str);
+
+ EXPECT_EQ(decoded, dyn);
+ } catch (const std::exception& err) {
+ LOG(ERROR) << err.what() << "\nInput: " << dyn.typeName() << ": " << dyn
+ << " decoded back as " << decoded.typeName() << ": " << decoded
+ << "\n" << folly::hexDump(str.data(), str.size());
+ throw;
+ }
+ }
+}
+
+TEST(Bser, Template) {
+ dynamic decoded(nullptr);
+ folly::fbstring str;
+ // Decode the template value provided from elsewhere
+ decoded = folly::bser::parseBser(
+ folly::ByteRange(template_blob, sizeof(template_blob) - 1));
+ EXPECT_EQ(decoded, template_dynamic)
+ << "Didn't load template value."
+ "\nInput: " << template_dynamic.typeName() << ": " << template_dynamic
+ << " decoded back as " << decoded.typeName() << ": " << decoded << "\n"
+ << folly::hexDump(template_blob, sizeof(template_blob) - 1);
+
+ // Now check that we can generate this same data representation
+ folly::bser::serialization_opts opts;
+ folly::bser::serialization_opts::TemplateMap templates = {
+ std::make_pair(&decoded, folly::dynamic{"name", "age"})};
+ opts.templates = templates;
+
+ str = folly::bser::toBser(decoded, opts);
+ EXPECT_EQ(folly::ByteRange((const uint8_t*)str.data(), str.size()),
+ folly::ByteRange(template_blob, sizeof(template_blob) - 1))
+ << "Expected:\n"
+ << folly::hexDump(template_blob, sizeof(template_blob) - 1) << "\nGot:\n"
+ << folly::hexDump(str.data(), str.size());
+}
+
+TEST(Bser, PduLength) {
+ EXPECT_THROW([] {
+ // Try to decode PDU for a short buffer that doesn't even have the
+ // complete length available
+ auto buf = folly::IOBuf::wrapBuffer(template_blob, 3);
+ auto len = folly::bser::decodePduLength(&*buf);
+ LOG(ERROR) << "managed to return a length, but only had 3 bytes";
+ }(), std::out_of_range);
+
+ auto buf = folly::IOBuf::wrapBuffer(template_blob, sizeof(template_blob));
+ auto len = folly::bser::decodePduLength(&*buf);
+ EXPECT_EQ(len, 44) << "PduLength should be 44, got " << len;
+}
+
+/* vim:ts=2:sw=2:et:
+ */
projects / folly.git / commitdiff