2 * Copyright 2013 Facebook, Inc.
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
17 #include "folly/io/Compression.h"
19 // Yes, tr1, as that's what gtest requires
23 #include <unordered_map>
25 #include <boost/noncopyable.hpp>
26 #include <glog/logging.h>
27 #include <gtest/gtest.h>
29 #include "folly/Benchmark.h"
30 #include "folly/Hash.h"
31 #include "folly/Random.h"
32 #include "folly/io/IOBufQueue.h"
34 namespace folly { namespace io { namespace test {
36 class DataHolder : private boost::noncopyable {
38 uint64_t hash(size_t size) const;
39 ByteRange data(size_t size) const;
42 explicit DataHolder(size_t sizeLog2);
44 std::unique_ptr<uint8_t[]> data_;
45 mutable std::unordered_map<uint64_t, uint64_t> hashCache_;
48 DataHolder::DataHolder(size_t sizeLog2)
49 : size_(size_t(1) << sizeLog2),
50 data_(new uint8_t[size_]) {
53 uint64_t DataHolder::hash(size_t size) const {
54 CHECK_LE(size, size_);
55 auto p = hashCache_.find(size);
56 if (p != hashCache_.end()) {
60 uint64_t h = folly::hash::fnv64_buf(data_.get(), size);
65 ByteRange DataHolder::data(size_t size) const {
66 CHECK_LE(size, size_);
67 return ByteRange(data_.get(), size);
70 uint64_t hashIOBuf(const IOBuf* buf) {
71 uint64_t h = folly::hash::FNV_64_HASH_START;
72 for (auto& range : *buf) {
73 h = folly::hash::fnv64_buf(range.data(), range.size(), h);
78 class RandomDataHolder : public DataHolder {
80 explicit RandomDataHolder(size_t sizeLog2);
83 RandomDataHolder::RandomDataHolder(size_t sizeLog2)
84 : DataHolder(sizeLog2) {
85 constexpr size_t numThreadsLog2 = 3;
86 constexpr size_t numThreads = size_t(1) << numThreadsLog2;
88 uint32_t seed = randomNumberSeed();
90 std::vector<std::thread> threads;
91 threads.reserve(numThreads);
92 for (size_t t = 0; t < numThreads; ++t) {
94 [this, seed, t, numThreadsLog2, sizeLog2] () {
95 std::mt19937 rng(seed + t);
96 size_t countLog2 = size_t(1) << (sizeLog2 - numThreadsLog2);
97 size_t start = size_t(t) << countLog2;
98 for (size_t i = 0; i < countLog2; ++i) {
99 this->data_[start + i] = rng();
104 for (auto& t : threads) {
109 class ConstantDataHolder : public DataHolder {
111 explicit ConstantDataHolder(size_t sizeLog2);
114 ConstantDataHolder::ConstantDataHolder(size_t sizeLog2)
115 : DataHolder(sizeLog2) {
116 memset(data_.get(), 'a', size_);
119 constexpr size_t dataSizeLog2 = 27; // 128MiB
120 RandomDataHolder randomDataHolder(dataSizeLog2);
121 ConstantDataHolder constantDataHolder(dataSizeLog2);
123 TEST(CompressionTestNeedsUncompressedLength, Simple) {
124 EXPECT_FALSE(getCodec(CodecType::NO_COMPRESSION)->needsUncompressedLength());
125 EXPECT_TRUE(getCodec(CodecType::LZ4)->needsUncompressedLength());
126 EXPECT_FALSE(getCodec(CodecType::SNAPPY)->needsUncompressedLength());
127 EXPECT_FALSE(getCodec(CodecType::ZLIB)->needsUncompressedLength());
128 EXPECT_FALSE(getCodec(CodecType::LZ4_VARINT_SIZE)->needsUncompressedLength());
131 class CompressionTest : public testing::TestWithParam<
132 std::tr1::tuple<int, CodecType>> {
135 auto tup = GetParam();
136 uncompressedLength_ = uint64_t(1) << std::tr1::get<0>(tup);
137 codec_ = getCodec(std::tr1::get<1>(tup));
140 void runSimpleTest(const DataHolder& dh);
142 uint64_t uncompressedLength_;
143 std::unique_ptr<Codec> codec_;
146 void CompressionTest::runSimpleTest(const DataHolder& dh) {
147 auto original = IOBuf::wrapBuffer(dh.data(uncompressedLength_));
148 auto compressed = codec_->compress(original.get());
149 if (!codec_->needsUncompressedLength()) {
150 auto uncompressed = codec_->uncompress(compressed.get());
151 EXPECT_EQ(uncompressedLength_, uncompressed->computeChainDataLength());
152 EXPECT_EQ(dh.hash(uncompressedLength_), hashIOBuf(uncompressed.get()));
155 auto uncompressed = codec_->uncompress(compressed.get(),
156 uncompressedLength_);
157 EXPECT_EQ(uncompressedLength_, uncompressed->computeChainDataLength());
158 EXPECT_EQ(dh.hash(uncompressedLength_), hashIOBuf(uncompressed.get()));
162 TEST_P(CompressionTest, RandomData) {
163 runSimpleTest(randomDataHolder);
166 TEST_P(CompressionTest, ConstantData) {
167 runSimpleTest(constantDataHolder);
170 INSTANTIATE_TEST_CASE_P(
174 testing::Values(0, 1, 12, 22, 25, 27),
175 testing::Values(CodecType::NO_COMPRESSION,
179 CodecType::LZ4_VARINT_SIZE)));
181 class CompressionCorruptionTest : public testing::TestWithParam<CodecType> {
184 codec_ = getCodec(GetParam());
187 void runSimpleTest(const DataHolder& dh);
189 std::unique_ptr<Codec> codec_;
192 void CompressionCorruptionTest::runSimpleTest(const DataHolder& dh) {
193 constexpr uint64_t uncompressedLength = 42;
194 auto original = IOBuf::wrapBuffer(dh.data(uncompressedLength));
195 auto compressed = codec_->compress(original.get());
197 if (!codec_->needsUncompressedLength()) {
198 auto uncompressed = codec_->uncompress(compressed.get());
199 EXPECT_EQ(uncompressedLength, uncompressed->computeChainDataLength());
200 EXPECT_EQ(dh.hash(uncompressedLength), hashIOBuf(uncompressed.get()));
203 auto uncompressed = codec_->uncompress(compressed.get(),
205 EXPECT_EQ(uncompressedLength, uncompressed->computeChainDataLength());
206 EXPECT_EQ(dh.hash(uncompressedLength), hashIOBuf(uncompressed.get()));
209 EXPECT_THROW(codec_->uncompress(compressed.get(), uncompressedLength + 1),
212 // Corrupt the first character
213 ++(compressed->writableData()[0]);
215 if (!codec_->needsUncompressedLength()) {
216 EXPECT_THROW(codec_->uncompress(compressed.get()),
220 EXPECT_THROW(codec_->uncompress(compressed.get(), uncompressedLength),
224 TEST_P(CompressionCorruptionTest, RandomData) {
225 runSimpleTest(randomDataHolder);
228 TEST_P(CompressionCorruptionTest, ConstantData) {
229 runSimpleTest(constantDataHolder);
232 INSTANTIATE_TEST_CASE_P(
233 CompressionCorruptionTest,
234 CompressionCorruptionTest,
236 // NO_COMPRESSION can't detect corruption
237 // LZ4 can't detect corruption reliably (sigh)
243 int main(int argc, char *argv[]) {
244 testing::InitGoogleTest(&argc, argv);
245 google::ParseCommandLineFlags(&argc, &argv, true);
247 auto ret = RUN_ALL_TESTS();
249 folly::runBenchmarksOnFlag();