2 * Copyright 2014 Facebook, Inc.
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
17 #include "folly/io/Compression.h"
24 #include <glog/logging.h>
26 #if FOLLY_HAVE_LIBSNAPPY
28 #include <snappy-sinksource.h>
35 #if FOLLY_HAVE_LIBLZMA
39 #include "folly/Conv.h"
40 #include "folly/Memory.h"
41 #include "folly/Portability.h"
42 #include "folly/ScopeGuard.h"
43 #include "folly/Varint.h"
44 #include "folly/io/Cursor.h"
46 namespace folly { namespace io {
48 Codec::Codec(CodecType type) : type_(type) { }
50 // Ensure consistent behavior in the nullptr case
51 std::unique_ptr<IOBuf> Codec::compress(const IOBuf* data) {
52 return !data->empty() ? doCompress(data) : IOBuf::create(0);
55 std::unique_ptr<IOBuf> Codec::uncompress(const IOBuf* data,
56 uint64_t uncompressedLength) {
57 if (uncompressedLength == UNKNOWN_UNCOMPRESSED_LENGTH) {
58 if (needsUncompressedLength()) {
59 throw std::invalid_argument("Codec: uncompressed length required");
61 } else if (uncompressedLength > maxUncompressedLength()) {
62 throw std::runtime_error("Codec: uncompressed length too large");
66 if (uncompressedLength != UNKNOWN_UNCOMPRESSED_LENGTH &&
67 uncompressedLength != 0) {
68 throw std::runtime_error("Codec: invalid uncompressed length");
70 return IOBuf::create(0);
73 return doUncompress(data, uncompressedLength);
76 bool Codec::needsUncompressedLength() const {
77 return doNeedsUncompressedLength();
80 uint64_t Codec::maxUncompressedLength() const {
81 return doMaxUncompressedLength();
84 bool Codec::doNeedsUncompressedLength() const {
88 uint64_t Codec::doMaxUncompressedLength() const {
89 return std::numeric_limits<uint64_t>::max() - 1;
97 class NoCompressionCodec FOLLY_FINAL : public Codec {
99 static std::unique_ptr<Codec> create(int level, CodecType type);
100 explicit NoCompressionCodec(int level, CodecType type);
103 std::unique_ptr<IOBuf> doCompress(const IOBuf* data) FOLLY_OVERRIDE;
104 std::unique_ptr<IOBuf> doUncompress(
106 uint64_t uncompressedLength) FOLLY_OVERRIDE;
109 std::unique_ptr<Codec> NoCompressionCodec::create(int level, CodecType type) {
110 return make_unique<NoCompressionCodec>(level, type);
113 NoCompressionCodec::NoCompressionCodec(int level, CodecType type)
115 DCHECK(type == CodecType::NO_COMPRESSION);
117 case COMPRESSION_LEVEL_DEFAULT:
118 case COMPRESSION_LEVEL_FASTEST:
119 case COMPRESSION_LEVEL_BEST:
123 throw std::invalid_argument(to<std::string>(
124 "NoCompressionCodec: invalid level ", level));
128 std::unique_ptr<IOBuf> NoCompressionCodec::doCompress(
130 return data->clone();
133 std::unique_ptr<IOBuf> NoCompressionCodec::doUncompress(
135 uint64_t uncompressedLength) {
136 if (uncompressedLength != UNKNOWN_UNCOMPRESSED_LENGTH &&
137 data->computeChainDataLength() != uncompressedLength) {
138 throw std::runtime_error(to<std::string>(
139 "NoCompressionCodec: invalid uncompressed length"));
141 return data->clone();
146 void encodeVarintToIOBuf(uint64_t val, folly::IOBuf* out) {
147 DCHECK_GE(out->tailroom(), kMaxVarintLength64);
148 out->append(encodeVarint(val, out->writableTail()));
151 uint64_t decodeVarintFromCursor(folly::io::Cursor& cursor) {
152 // Must have enough room in *this* buffer.
153 auto p = cursor.peek();
154 folly::ByteRange range(p.first, p.second);
155 uint64_t val = decodeVarint(range);
156 cursor.skip(range.data() - p.first);
162 #if FOLLY_HAVE_LIBLZ4
167 class LZ4Codec FOLLY_FINAL : public Codec {
169 static std::unique_ptr<Codec> create(int level, CodecType type);
170 explicit LZ4Codec(int level, CodecType type);
173 bool doNeedsUncompressedLength() const FOLLY_OVERRIDE;
174 uint64_t doMaxUncompressedLength() const FOLLY_OVERRIDE;
176 bool encodeSize() const { return type() == CodecType::LZ4_VARINT_SIZE; }
178 std::unique_ptr<IOBuf> doCompress(const IOBuf* data) FOLLY_OVERRIDE;
179 std::unique_ptr<IOBuf> doUncompress(
181 uint64_t uncompressedLength) FOLLY_OVERRIDE;
183 bool highCompression_;
186 std::unique_ptr<Codec> LZ4Codec::create(int level, CodecType type) {
187 return make_unique<LZ4Codec>(level, type);
190 LZ4Codec::LZ4Codec(int level, CodecType type) : Codec(type) {
191 DCHECK(type == CodecType::LZ4 || type == CodecType::LZ4_VARINT_SIZE);
194 case COMPRESSION_LEVEL_FASTEST:
195 case COMPRESSION_LEVEL_DEFAULT:
198 case COMPRESSION_LEVEL_BEST:
202 if (level < 1 || level > 2) {
203 throw std::invalid_argument(to<std::string>(
204 "LZ4Codec: invalid level: ", level));
206 highCompression_ = (level > 1);
209 bool LZ4Codec::doNeedsUncompressedLength() const {
210 return !encodeSize();
213 uint64_t LZ4Codec::doMaxUncompressedLength() const {
214 // From lz4.h: "Max supported value is ~1.9GB"; I wish we had something
216 return 1.8 * (uint64_t(1) << 30);
219 std::unique_ptr<IOBuf> LZ4Codec::doCompress(const IOBuf* data) {
220 std::unique_ptr<IOBuf> clone;
221 if (data->isChained()) {
222 // LZ4 doesn't support streaming, so we have to coalesce
223 clone = data->clone();
228 uint32_t extraSize = encodeSize() ? kMaxVarintLength64 : 0;
229 auto out = IOBuf::create(extraSize + LZ4_compressBound(data->length()));
231 encodeVarintToIOBuf(data->length(), out.get());
235 if (highCompression_) {
236 n = LZ4_compressHC(reinterpret_cast<const char*>(data->data()),
237 reinterpret_cast<char*>(out->writableTail()),
240 n = LZ4_compress(reinterpret_cast<const char*>(data->data()),
241 reinterpret_cast<char*>(out->writableTail()),
246 CHECK_LE(n, out->capacity());
252 std::unique_ptr<IOBuf> LZ4Codec::doUncompress(
254 uint64_t uncompressedLength) {
255 std::unique_ptr<IOBuf> clone;
256 if (data->isChained()) {
257 // LZ4 doesn't support streaming, so we have to coalesce
258 clone = data->clone();
263 folly::io::Cursor cursor(data);
264 uint64_t actualUncompressedLength;
266 actualUncompressedLength = decodeVarintFromCursor(cursor);
267 if (uncompressedLength != UNKNOWN_UNCOMPRESSED_LENGTH &&
268 uncompressedLength != actualUncompressedLength) {
269 throw std::runtime_error("LZ4Codec: invalid uncompressed length");
272 actualUncompressedLength = uncompressedLength;
273 DCHECK_NE(actualUncompressedLength, UNKNOWN_UNCOMPRESSED_LENGTH);
276 auto out = IOBuf::create(actualUncompressedLength);
277 auto p = cursor.peek();
278 int n = LZ4_uncompress(reinterpret_cast<const char*>(p.first),
279 reinterpret_cast<char*>(out->writableTail()),
280 actualUncompressedLength);
282 throw std::runtime_error(to<std::string>(
283 "LZ4 decompression returned invalid value ", n));
285 out->append(actualUncompressedLength);
289 #endif // FOLLY_HAVE_LIBLZ4
291 #if FOLLY_HAVE_LIBSNAPPY
298 * Implementation of snappy::Source that reads from a IOBuf chain.
300 class IOBufSnappySource FOLLY_FINAL : public snappy::Source {
302 explicit IOBufSnappySource(const IOBuf* data);
303 size_t Available() const FOLLY_OVERRIDE;
304 const char* Peek(size_t* len) FOLLY_OVERRIDE;
305 void Skip(size_t n) FOLLY_OVERRIDE;
311 IOBufSnappySource::IOBufSnappySource(const IOBuf* data)
312 : available_(data->computeChainDataLength()),
316 size_t IOBufSnappySource::Available() const {
320 const char* IOBufSnappySource::Peek(size_t* len) {
321 auto p = cursor_.peek();
323 return reinterpret_cast<const char*>(p.first);
326 void IOBufSnappySource::Skip(size_t n) {
327 CHECK_LE(n, available_);
332 class SnappyCodec FOLLY_FINAL : public Codec {
334 static std::unique_ptr<Codec> create(int level, CodecType type);
335 explicit SnappyCodec(int level, CodecType type);
338 uint64_t doMaxUncompressedLength() const FOLLY_OVERRIDE;
339 std::unique_ptr<IOBuf> doCompress(const IOBuf* data) FOLLY_OVERRIDE;
340 std::unique_ptr<IOBuf> doUncompress(
342 uint64_t uncompressedLength) FOLLY_OVERRIDE;
345 std::unique_ptr<Codec> SnappyCodec::create(int level, CodecType type) {
346 return make_unique<SnappyCodec>(level, type);
349 SnappyCodec::SnappyCodec(int level, CodecType type) : Codec(type) {
350 DCHECK(type == CodecType::SNAPPY);
352 case COMPRESSION_LEVEL_FASTEST:
353 case COMPRESSION_LEVEL_DEFAULT:
354 case COMPRESSION_LEVEL_BEST:
358 throw std::invalid_argument(to<std::string>(
359 "SnappyCodec: invalid level: ", level));
363 uint64_t SnappyCodec::doMaxUncompressedLength() const {
364 // snappy.h uses uint32_t for lengths, so there's that.
365 return std::numeric_limits<uint32_t>::max();
368 std::unique_ptr<IOBuf> SnappyCodec::doCompress(const IOBuf* data) {
369 IOBufSnappySource source(data);
371 IOBuf::create(snappy::MaxCompressedLength(source.Available()));
373 snappy::UncheckedByteArraySink sink(reinterpret_cast<char*>(
374 out->writableTail()));
376 size_t n = snappy::Compress(&source, &sink);
378 CHECK_LE(n, out->capacity());
383 std::unique_ptr<IOBuf> SnappyCodec::doUncompress(const IOBuf* data,
384 uint64_t uncompressedLength) {
385 uint32_t actualUncompressedLength = 0;
388 IOBufSnappySource source(data);
389 if (!snappy::GetUncompressedLength(&source, &actualUncompressedLength)) {
390 throw std::runtime_error("snappy::GetUncompressedLength failed");
392 if (uncompressedLength != UNKNOWN_UNCOMPRESSED_LENGTH &&
393 uncompressedLength != actualUncompressedLength) {
394 throw std::runtime_error("snappy: invalid uncompressed length");
398 auto out = IOBuf::create(actualUncompressedLength);
401 IOBufSnappySource source(data);
402 if (!snappy::RawUncompress(&source,
403 reinterpret_cast<char*>(out->writableTail()))) {
404 throw std::runtime_error("snappy::RawUncompress failed");
408 out->append(actualUncompressedLength);
412 #endif // FOLLY_HAVE_LIBSNAPPY
418 class ZlibCodec FOLLY_FINAL : public Codec {
420 static std::unique_ptr<Codec> create(int level, CodecType type);
421 explicit ZlibCodec(int level, CodecType type);
424 std::unique_ptr<IOBuf> doCompress(const IOBuf* data) FOLLY_OVERRIDE;
425 std::unique_ptr<IOBuf> doUncompress(
427 uint64_t uncompressedLength) FOLLY_OVERRIDE;
429 std::unique_ptr<IOBuf> addOutputBuffer(z_stream* stream, uint32_t length);
430 bool doInflate(z_stream* stream, IOBuf* head, uint32_t bufferLength);
435 std::unique_ptr<Codec> ZlibCodec::create(int level, CodecType type) {
436 return make_unique<ZlibCodec>(level, type);
439 ZlibCodec::ZlibCodec(int level, CodecType type) : Codec(type) {
440 DCHECK(type == CodecType::ZLIB);
442 case COMPRESSION_LEVEL_FASTEST:
445 case COMPRESSION_LEVEL_DEFAULT:
446 level = Z_DEFAULT_COMPRESSION;
448 case COMPRESSION_LEVEL_BEST:
452 if (level != Z_DEFAULT_COMPRESSION && (level < 0 || level > 9)) {
453 throw std::invalid_argument(to<std::string>(
454 "ZlibCodec: invalid level: ", level));
459 std::unique_ptr<IOBuf> ZlibCodec::addOutputBuffer(z_stream* stream,
461 CHECK_EQ(stream->avail_out, 0);
463 auto buf = IOBuf::create(length);
466 stream->next_out = buf->writableData();
467 stream->avail_out = buf->length();
472 bool ZlibCodec::doInflate(z_stream* stream,
474 uint32_t bufferLength) {
475 if (stream->avail_out == 0) {
476 head->prependChain(addOutputBuffer(stream, bufferLength));
479 int rc = inflate(stream, Z_NO_FLUSH);
490 throw std::runtime_error(to<std::string>(
491 "ZlibCodec: inflate error: ", rc, ": ", stream->msg));
493 CHECK(false) << rc << ": " << stream->msg;
499 std::unique_ptr<IOBuf> ZlibCodec::doCompress(const IOBuf* data) {
501 stream.zalloc = nullptr;
502 stream.zfree = nullptr;
503 stream.opaque = nullptr;
505 int rc = deflateInit(&stream, level_);
507 throw std::runtime_error(to<std::string>(
508 "ZlibCodec: deflateInit error: ", rc, ": ", stream.msg));
511 stream.next_in = stream.next_out = nullptr;
512 stream.avail_in = stream.avail_out = 0;
513 stream.total_in = stream.total_out = 0;
515 bool success = false;
518 int rc = deflateEnd(&stream);
519 // If we're here because of an exception, it's okay if some data
521 CHECK(rc == Z_OK || (!success && rc == Z_DATA_ERROR))
522 << rc << ": " << stream.msg;
525 uint64_t uncompressedLength = data->computeChainDataLength();
526 uint64_t maxCompressedLength = deflateBound(&stream, uncompressedLength);
528 // Max 64MiB in one go
529 constexpr uint32_t maxSingleStepLength = uint32_t(64) << 20; // 64MiB
530 constexpr uint32_t defaultBufferLength = uint32_t(4) << 20; // 4MiB
532 auto out = addOutputBuffer(
534 (maxCompressedLength <= maxSingleStepLength ?
535 maxCompressedLength :
536 defaultBufferLength));
538 for (auto& range : *data) {
543 stream.next_in = const_cast<uint8_t*>(range.data());
544 stream.avail_in = range.size();
546 while (stream.avail_in != 0) {
547 if (stream.avail_out == 0) {
548 out->prependChain(addOutputBuffer(&stream, defaultBufferLength));
551 rc = deflate(&stream, Z_NO_FLUSH);
553 CHECK_EQ(rc, Z_OK) << stream.msg;
558 if (stream.avail_out == 0) {
559 out->prependChain(addOutputBuffer(&stream, defaultBufferLength));
562 rc = deflate(&stream, Z_FINISH);
563 } while (rc == Z_OK);
565 CHECK_EQ(rc, Z_STREAM_END) << stream.msg;
567 out->prev()->trimEnd(stream.avail_out);
569 success = true; // we survived
574 std::unique_ptr<IOBuf> ZlibCodec::doUncompress(const IOBuf* data,
575 uint64_t uncompressedLength) {
577 stream.zalloc = nullptr;
578 stream.zfree = nullptr;
579 stream.opaque = nullptr;
581 int rc = inflateInit(&stream);
583 throw std::runtime_error(to<std::string>(
584 "ZlibCodec: inflateInit error: ", rc, ": ", stream.msg));
587 stream.next_in = stream.next_out = nullptr;
588 stream.avail_in = stream.avail_out = 0;
589 stream.total_in = stream.total_out = 0;
591 bool success = false;
594 int rc = inflateEnd(&stream);
595 // If we're here because of an exception, it's okay if some data
597 CHECK(rc == Z_OK || (!success && rc == Z_DATA_ERROR))
598 << rc << ": " << stream.msg;
601 // Max 64MiB in one go
602 constexpr uint32_t maxSingleStepLength = uint32_t(64) << 20; // 64MiB
603 constexpr uint32_t defaultBufferLength = uint32_t(4) << 20; // 4MiB
605 auto out = addOutputBuffer(
607 ((uncompressedLength != UNKNOWN_UNCOMPRESSED_LENGTH &&
608 uncompressedLength <= maxSingleStepLength) ?
610 defaultBufferLength));
612 bool streamEnd = false;
613 for (auto& range : *data) {
618 stream.next_in = const_cast<uint8_t*>(range.data());
619 stream.avail_in = range.size();
621 while (stream.avail_in != 0) {
623 throw std::runtime_error(to<std::string>(
624 "ZlibCodec: junk after end of data"));
627 streamEnd = doInflate(&stream, out.get(), defaultBufferLength);
632 streamEnd = doInflate(&stream, out.get(), defaultBufferLength);
635 out->prev()->trimEnd(stream.avail_out);
637 if (uncompressedLength != UNKNOWN_UNCOMPRESSED_LENGTH &&
638 uncompressedLength != stream.total_out) {
639 throw std::runtime_error(to<std::string>(
640 "ZlibCodec: invalid uncompressed length"));
643 success = true; // we survived
648 #endif // FOLLY_HAVE_LIBZ
650 #if FOLLY_HAVE_LIBLZMA
655 class LZMA2Codec FOLLY_FINAL : public Codec {
657 static std::unique_ptr<Codec> create(int level, CodecType type);
658 explicit LZMA2Codec(int level, CodecType type);
661 bool doNeedsUncompressedLength() const FOLLY_OVERRIDE;
662 uint64_t doMaxUncompressedLength() const FOLLY_OVERRIDE;
664 bool encodeSize() const { return type() == CodecType::LZMA2_VARINT_SIZE; }
666 std::unique_ptr<IOBuf> doCompress(const IOBuf* data) FOLLY_OVERRIDE;
667 std::unique_ptr<IOBuf> doUncompress(
669 uint64_t uncompressedLength) FOLLY_OVERRIDE;
671 std::unique_ptr<IOBuf> addOutputBuffer(lzma_stream* stream, size_t length);
672 bool doInflate(lzma_stream* stream, IOBuf* head, size_t bufferLength);
677 std::unique_ptr<Codec> LZMA2Codec::create(int level, CodecType type) {
678 return make_unique<LZMA2Codec>(level, type);
681 LZMA2Codec::LZMA2Codec(int level, CodecType type) : Codec(type) {
682 DCHECK(type == CodecType::LZMA2 || type == CodecType::LZMA2_VARINT_SIZE);
684 case COMPRESSION_LEVEL_FASTEST:
687 case COMPRESSION_LEVEL_DEFAULT:
688 level = LZMA_PRESET_DEFAULT;
690 case COMPRESSION_LEVEL_BEST:
694 if (level < 0 || level > 9) {
695 throw std::invalid_argument(to<std::string>(
696 "LZMA2Codec: invalid level: ", level));
701 bool LZMA2Codec::doNeedsUncompressedLength() const {
702 return !encodeSize();
705 uint64_t LZMA2Codec::doMaxUncompressedLength() const {
706 // From lzma/base.h: "Stream is roughly 8 EiB (2^63 bytes)"
707 return uint64_t(1) << 63;
710 std::unique_ptr<IOBuf> LZMA2Codec::addOutputBuffer(
714 CHECK_EQ(stream->avail_out, 0);
716 auto buf = IOBuf::create(length);
719 stream->next_out = buf->writableData();
720 stream->avail_out = buf->length();
725 std::unique_ptr<IOBuf> LZMA2Codec::doCompress(const IOBuf* data) {
727 lzma_stream stream = LZMA_STREAM_INIT;
729 rc = lzma_easy_encoder(&stream, level_, LZMA_CHECK_NONE);
731 throw std::runtime_error(folly::to<std::string>(
732 "LZMA2Codec: lzma_easy_encoder error: ", rc));
735 SCOPE_EXIT { lzma_end(&stream); };
737 uint64_t uncompressedLength = data->computeChainDataLength();
738 uint64_t maxCompressedLength = lzma_stream_buffer_bound(uncompressedLength);
740 // Max 64MiB in one go
741 constexpr uint32_t maxSingleStepLength = uint32_t(64) << 20; // 64MiB
742 constexpr uint32_t defaultBufferLength = uint32_t(4) << 20; // 4MiB
744 auto out = addOutputBuffer(
746 (maxCompressedLength <= maxSingleStepLength ?
747 maxCompressedLength :
748 defaultBufferLength));
751 auto size = IOBuf::createCombined(kMaxVarintLength64);
752 encodeVarintToIOBuf(uncompressedLength, size.get());
753 size->appendChain(std::move(out));
754 out = std::move(size);
757 for (auto& range : *data) {
762 stream.next_in = const_cast<uint8_t*>(range.data());
763 stream.avail_in = range.size();
765 while (stream.avail_in != 0) {
766 if (stream.avail_out == 0) {
767 out->prependChain(addOutputBuffer(&stream, defaultBufferLength));
770 rc = lzma_code(&stream, LZMA_RUN);
773 throw std::runtime_error(folly::to<std::string>(
774 "LZMA2Codec: lzma_code error: ", rc));
780 if (stream.avail_out == 0) {
781 out->prependChain(addOutputBuffer(&stream, defaultBufferLength));
784 rc = lzma_code(&stream, LZMA_FINISH);
785 } while (rc == LZMA_OK);
787 if (rc != LZMA_STREAM_END) {
788 throw std::runtime_error(folly::to<std::string>(
789 "LZMA2Codec: lzma_code ended with error: ", rc));
792 out->prev()->trimEnd(stream.avail_out);
797 bool LZMA2Codec::doInflate(lzma_stream* stream,
799 size_t bufferLength) {
800 if (stream->avail_out == 0) {
801 head->prependChain(addOutputBuffer(stream, bufferLength));
804 lzma_ret rc = lzma_code(stream, LZMA_RUN);
809 case LZMA_STREAM_END:
812 throw std::runtime_error(to<std::string>(
813 "LZMA2Codec: lzma_code error: ", rc));
819 std::unique_ptr<IOBuf> LZMA2Codec::doUncompress(const IOBuf* data,
820 uint64_t uncompressedLength) {
822 lzma_stream stream = LZMA_STREAM_INIT;
824 rc = lzma_auto_decoder(&stream, std::numeric_limits<uint64_t>::max(), 0);
826 throw std::runtime_error(folly::to<std::string>(
827 "LZMA2Codec: lzma_auto_decoder error: ", rc));
830 SCOPE_EXIT { lzma_end(&stream); };
832 // Max 64MiB in one go
833 constexpr uint32_t maxSingleStepLength = uint32_t(64) << 20; // 64MiB
834 constexpr uint32_t defaultBufferLength = uint32_t(4) << 20; // 4MiB
836 folly::io::Cursor cursor(data);
837 uint64_t actualUncompressedLength;
839 actualUncompressedLength = decodeVarintFromCursor(cursor);
840 if (uncompressedLength != UNKNOWN_UNCOMPRESSED_LENGTH &&
841 uncompressedLength != actualUncompressedLength) {
842 throw std::runtime_error("LZMA2Codec: invalid uncompressed length");
845 actualUncompressedLength = uncompressedLength;
846 DCHECK_NE(actualUncompressedLength, UNKNOWN_UNCOMPRESSED_LENGTH);
849 auto out = addOutputBuffer(
851 (actualUncompressedLength <= maxSingleStepLength ?
852 actualUncompressedLength :
853 defaultBufferLength));
855 bool streamEnd = false;
856 auto buf = cursor.peek();
857 while (buf.second != 0) {
858 stream.next_in = const_cast<uint8_t*>(buf.first);
859 stream.avail_in = buf.second;
861 while (stream.avail_in != 0) {
863 throw std::runtime_error(to<std::string>(
864 "LZMA2Codec: junk after end of data"));
867 streamEnd = doInflate(&stream, out.get(), defaultBufferLength);
870 cursor.skip(buf.second);
875 streamEnd = doInflate(&stream, out.get(), defaultBufferLength);
878 out->prev()->trimEnd(stream.avail_out);
880 if (actualUncompressedLength != stream.total_out) {
881 throw std::runtime_error(to<std::string>(
882 "LZMA2Codec: invalid uncompressed length"));
888 #endif // FOLLY_HAVE_LIBLZMA
890 typedef std::unique_ptr<Codec> (*CodecFactory)(int, CodecType);
892 CodecFactory gCodecFactories[
893 static_cast<size_t>(CodecType::NUM_CODEC_TYPES)] = {
894 nullptr, // USER_DEFINED
895 NoCompressionCodec::create,
897 #if FOLLY_HAVE_LIBLZ4
903 #if FOLLY_HAVE_LIBSNAPPY
915 #if FOLLY_HAVE_LIBLZ4
921 #if FOLLY_HAVE_LIBLZMA
932 std::unique_ptr<Codec> getCodec(CodecType type, int level) {
933 size_t idx = static_cast<size_t>(type);
934 if (idx >= static_cast<size_t>(CodecType::NUM_CODEC_TYPES)) {
935 throw std::invalid_argument(to<std::string>(
936 "Compression type ", idx, " not supported"));
938 auto factory = gCodecFactories[idx];
940 throw std::invalid_argument(to<std::string>(
941 "Compression type ", idx, " not supported"));
943 auto codec = (*factory)(level, type);
944 DCHECK_EQ(static_cast<size_t>(codec->type()), idx);