2 * Copyright 2014 Facebook, Inc.
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
17 #include <folly/io/Compression.h>
24 #include <glog/logging.h>
26 #if FOLLY_HAVE_LIBSNAPPY
28 #include <snappy-sinksource.h>
35 #if FOLLY_HAVE_LIBLZMA
39 #include <folly/Conv.h>
40 #include <folly/Memory.h>
41 #include <folly/Portability.h>
42 #include <folly/ScopeGuard.h>
43 #include <folly/Varint.h>
44 #include <folly/io/Cursor.h>
46 namespace folly { namespace io {
48 Codec::Codec(CodecType type) : type_(type) { }
50 // Ensure consistent behavior in the nullptr case
51 std::unique_ptr<IOBuf> Codec::compress(const IOBuf* data) {
52 uint64_t len = data->computeChainDataLength();
54 return IOBuf::create(0);
55 } else if (len > maxUncompressedLength()) {
56 throw std::runtime_error("Codec: uncompressed length too large");
59 return doCompress(data);
62 std::unique_ptr<IOBuf> Codec::uncompress(const IOBuf* data,
63 uint64_t uncompressedLength) {
64 if (uncompressedLength == UNKNOWN_UNCOMPRESSED_LENGTH) {
65 if (needsUncompressedLength()) {
66 throw std::invalid_argument("Codec: uncompressed length required");
68 } else if (uncompressedLength > maxUncompressedLength()) {
69 throw std::runtime_error("Codec: uncompressed length too large");
73 if (uncompressedLength != UNKNOWN_UNCOMPRESSED_LENGTH &&
74 uncompressedLength != 0) {
75 throw std::runtime_error("Codec: invalid uncompressed length");
77 return IOBuf::create(0);
80 return doUncompress(data, uncompressedLength);
83 bool Codec::needsUncompressedLength() const {
84 return doNeedsUncompressedLength();
87 uint64_t Codec::maxUncompressedLength() const {
88 return doMaxUncompressedLength();
91 bool Codec::doNeedsUncompressedLength() const {
95 uint64_t Codec::doMaxUncompressedLength() const {
96 return UNLIMITED_UNCOMPRESSED_LENGTH;
104 class NoCompressionCodec FOLLY_FINAL : public Codec {
106 static std::unique_ptr<Codec> create(int level, CodecType type);
107 explicit NoCompressionCodec(int level, CodecType type);
110 std::unique_ptr<IOBuf> doCompress(const IOBuf* data) FOLLY_OVERRIDE;
111 std::unique_ptr<IOBuf> doUncompress(
113 uint64_t uncompressedLength) FOLLY_OVERRIDE;
116 std::unique_ptr<Codec> NoCompressionCodec::create(int level, CodecType type) {
117 return make_unique<NoCompressionCodec>(level, type);
120 NoCompressionCodec::NoCompressionCodec(int level, CodecType type)
122 DCHECK(type == CodecType::NO_COMPRESSION);
124 case COMPRESSION_LEVEL_DEFAULT:
125 case COMPRESSION_LEVEL_FASTEST:
126 case COMPRESSION_LEVEL_BEST:
130 throw std::invalid_argument(to<std::string>(
131 "NoCompressionCodec: invalid level ", level));
135 std::unique_ptr<IOBuf> NoCompressionCodec::doCompress(
137 return data->clone();
140 std::unique_ptr<IOBuf> NoCompressionCodec::doUncompress(
142 uint64_t uncompressedLength) {
143 if (uncompressedLength != UNKNOWN_UNCOMPRESSED_LENGTH &&
144 data->computeChainDataLength() != uncompressedLength) {
145 throw std::runtime_error(to<std::string>(
146 "NoCompressionCodec: invalid uncompressed length"));
148 return data->clone();
153 void encodeVarintToIOBuf(uint64_t val, folly::IOBuf* out) {
154 DCHECK_GE(out->tailroom(), kMaxVarintLength64);
155 out->append(encodeVarint(val, out->writableTail()));
158 inline uint64_t decodeVarintFromCursor(folly::io::Cursor& cursor) {
161 for (int shift = 0; shift <= 63; shift += 7) {
162 b = cursor.read<int8_t>();
163 val |= static_cast<uint64_t>(b & 0x7f) << shift;
169 throw std::invalid_argument("Invalid varint value. Too big.");
176 #if FOLLY_HAVE_LIBLZ4
181 class LZ4Codec FOLLY_FINAL : public Codec {
183 static std::unique_ptr<Codec> create(int level, CodecType type);
184 explicit LZ4Codec(int level, CodecType type);
187 bool doNeedsUncompressedLength() const FOLLY_OVERRIDE;
188 uint64_t doMaxUncompressedLength() const FOLLY_OVERRIDE;
190 bool encodeSize() const { return type() == CodecType::LZ4_VARINT_SIZE; }
192 std::unique_ptr<IOBuf> doCompress(const IOBuf* data) FOLLY_OVERRIDE;
193 std::unique_ptr<IOBuf> doUncompress(
195 uint64_t uncompressedLength) FOLLY_OVERRIDE;
197 bool highCompression_;
200 std::unique_ptr<Codec> LZ4Codec::create(int level, CodecType type) {
201 return make_unique<LZ4Codec>(level, type);
204 LZ4Codec::LZ4Codec(int level, CodecType type) : Codec(type) {
205 DCHECK(type == CodecType::LZ4 || type == CodecType::LZ4_VARINT_SIZE);
208 case COMPRESSION_LEVEL_FASTEST:
209 case COMPRESSION_LEVEL_DEFAULT:
212 case COMPRESSION_LEVEL_BEST:
216 if (level < 1 || level > 2) {
217 throw std::invalid_argument(to<std::string>(
218 "LZ4Codec: invalid level: ", level));
220 highCompression_ = (level > 1);
223 bool LZ4Codec::doNeedsUncompressedLength() const {
224 return !encodeSize();
227 // The value comes from lz4.h in lz4-r117, but older versions of lz4 don't
228 // define LZ4_MAX_INPUT_SIZE (even though the max size is the same), so do it
230 #ifndef LZ4_MAX_INPUT_SIZE
231 # define LZ4_MAX_INPUT_SIZE 0x7E000000
234 uint64_t LZ4Codec::doMaxUncompressedLength() const {
235 return LZ4_MAX_INPUT_SIZE;
238 std::unique_ptr<IOBuf> LZ4Codec::doCompress(const IOBuf* data) {
239 std::unique_ptr<IOBuf> clone;
240 if (data->isChained()) {
241 // LZ4 doesn't support streaming, so we have to coalesce
242 clone = data->clone();
247 uint32_t extraSize = encodeSize() ? kMaxVarintLength64 : 0;
248 auto out = IOBuf::create(extraSize + LZ4_compressBound(data->length()));
250 encodeVarintToIOBuf(data->length(), out.get());
254 if (highCompression_) {
255 n = LZ4_compressHC(reinterpret_cast<const char*>(data->data()),
256 reinterpret_cast<char*>(out->writableTail()),
259 n = LZ4_compress(reinterpret_cast<const char*>(data->data()),
260 reinterpret_cast<char*>(out->writableTail()),
265 CHECK_LE(n, out->capacity());
271 std::unique_ptr<IOBuf> LZ4Codec::doUncompress(
273 uint64_t uncompressedLength) {
274 std::unique_ptr<IOBuf> clone;
275 if (data->isChained()) {
276 // LZ4 doesn't support streaming, so we have to coalesce
277 clone = data->clone();
282 folly::io::Cursor cursor(data);
283 uint64_t actualUncompressedLength;
285 actualUncompressedLength = decodeVarintFromCursor(cursor);
286 if (uncompressedLength != UNKNOWN_UNCOMPRESSED_LENGTH &&
287 uncompressedLength != actualUncompressedLength) {
288 throw std::runtime_error("LZ4Codec: invalid uncompressed length");
291 actualUncompressedLength = uncompressedLength;
292 if (actualUncompressedLength == UNKNOWN_UNCOMPRESSED_LENGTH ||
293 actualUncompressedLength > maxUncompressedLength()) {
294 throw std::runtime_error("LZ4Codec: invalid uncompressed length");
298 auto p = cursor.peek();
299 auto out = IOBuf::create(actualUncompressedLength);
300 int n = LZ4_decompress_safe(reinterpret_cast<const char*>(p.first),
301 reinterpret_cast<char*>(out->writableTail()),
303 actualUncompressedLength);
305 if (n < 0 || uint64_t(n) != actualUncompressedLength) {
306 throw std::runtime_error(to<std::string>(
307 "LZ4 decompression returned invalid value ", n));
309 out->append(actualUncompressedLength);
313 #endif // FOLLY_HAVE_LIBLZ4
315 #if FOLLY_HAVE_LIBSNAPPY
322 * Implementation of snappy::Source that reads from a IOBuf chain.
324 class IOBufSnappySource FOLLY_FINAL : public snappy::Source {
326 explicit IOBufSnappySource(const IOBuf* data);
327 size_t Available() const FOLLY_OVERRIDE;
328 const char* Peek(size_t* len) FOLLY_OVERRIDE;
329 void Skip(size_t n) FOLLY_OVERRIDE;
335 IOBufSnappySource::IOBufSnappySource(const IOBuf* data)
336 : available_(data->computeChainDataLength()),
340 size_t IOBufSnappySource::Available() const {
344 const char* IOBufSnappySource::Peek(size_t* len) {
345 auto p = cursor_.peek();
347 return reinterpret_cast<const char*>(p.first);
350 void IOBufSnappySource::Skip(size_t n) {
351 CHECK_LE(n, available_);
356 class SnappyCodec FOLLY_FINAL : public Codec {
358 static std::unique_ptr<Codec> create(int level, CodecType type);
359 explicit SnappyCodec(int level, CodecType type);
362 uint64_t doMaxUncompressedLength() const FOLLY_OVERRIDE;
363 std::unique_ptr<IOBuf> doCompress(const IOBuf* data) FOLLY_OVERRIDE;
364 std::unique_ptr<IOBuf> doUncompress(
366 uint64_t uncompressedLength) FOLLY_OVERRIDE;
369 std::unique_ptr<Codec> SnappyCodec::create(int level, CodecType type) {
370 return make_unique<SnappyCodec>(level, type);
373 SnappyCodec::SnappyCodec(int level, CodecType type) : Codec(type) {
374 DCHECK(type == CodecType::SNAPPY);
376 case COMPRESSION_LEVEL_FASTEST:
377 case COMPRESSION_LEVEL_DEFAULT:
378 case COMPRESSION_LEVEL_BEST:
382 throw std::invalid_argument(to<std::string>(
383 "SnappyCodec: invalid level: ", level));
387 uint64_t SnappyCodec::doMaxUncompressedLength() const {
388 // snappy.h uses uint32_t for lengths, so there's that.
389 return std::numeric_limits<uint32_t>::max();
392 std::unique_ptr<IOBuf> SnappyCodec::doCompress(const IOBuf* data) {
393 IOBufSnappySource source(data);
395 IOBuf::create(snappy::MaxCompressedLength(source.Available()));
397 snappy::UncheckedByteArraySink sink(reinterpret_cast<char*>(
398 out->writableTail()));
400 size_t n = snappy::Compress(&source, &sink);
402 CHECK_LE(n, out->capacity());
407 std::unique_ptr<IOBuf> SnappyCodec::doUncompress(const IOBuf* data,
408 uint64_t uncompressedLength) {
409 uint32_t actualUncompressedLength = 0;
412 IOBufSnappySource source(data);
413 if (!snappy::GetUncompressedLength(&source, &actualUncompressedLength)) {
414 throw std::runtime_error("snappy::GetUncompressedLength failed");
416 if (uncompressedLength != UNKNOWN_UNCOMPRESSED_LENGTH &&
417 uncompressedLength != actualUncompressedLength) {
418 throw std::runtime_error("snappy: invalid uncompressed length");
422 auto out = IOBuf::create(actualUncompressedLength);
425 IOBufSnappySource source(data);
426 if (!snappy::RawUncompress(&source,
427 reinterpret_cast<char*>(out->writableTail()))) {
428 throw std::runtime_error("snappy::RawUncompress failed");
432 out->append(actualUncompressedLength);
436 #endif // FOLLY_HAVE_LIBSNAPPY
442 class ZlibCodec FOLLY_FINAL : public Codec {
444 static std::unique_ptr<Codec> create(int level, CodecType type);
445 explicit ZlibCodec(int level, CodecType type);
448 std::unique_ptr<IOBuf> doCompress(const IOBuf* data) FOLLY_OVERRIDE;
449 std::unique_ptr<IOBuf> doUncompress(
451 uint64_t uncompressedLength) FOLLY_OVERRIDE;
453 std::unique_ptr<IOBuf> addOutputBuffer(z_stream* stream, uint32_t length);
454 bool doInflate(z_stream* stream, IOBuf* head, uint32_t bufferLength);
459 std::unique_ptr<Codec> ZlibCodec::create(int level, CodecType type) {
460 return make_unique<ZlibCodec>(level, type);
463 ZlibCodec::ZlibCodec(int level, CodecType type) : Codec(type) {
464 DCHECK(type == CodecType::ZLIB);
466 case COMPRESSION_LEVEL_FASTEST:
469 case COMPRESSION_LEVEL_DEFAULT:
470 level = Z_DEFAULT_COMPRESSION;
472 case COMPRESSION_LEVEL_BEST:
476 if (level != Z_DEFAULT_COMPRESSION && (level < 0 || level > 9)) {
477 throw std::invalid_argument(to<std::string>(
478 "ZlibCodec: invalid level: ", level));
483 std::unique_ptr<IOBuf> ZlibCodec::addOutputBuffer(z_stream* stream,
485 CHECK_EQ(stream->avail_out, 0);
487 auto buf = IOBuf::create(length);
490 stream->next_out = buf->writableData();
491 stream->avail_out = buf->length();
496 bool ZlibCodec::doInflate(z_stream* stream,
498 uint32_t bufferLength) {
499 if (stream->avail_out == 0) {
500 head->prependChain(addOutputBuffer(stream, bufferLength));
503 int rc = inflate(stream, Z_NO_FLUSH);
514 throw std::runtime_error(to<std::string>(
515 "ZlibCodec: inflate error: ", rc, ": ", stream->msg));
517 CHECK(false) << rc << ": " << stream->msg;
523 std::unique_ptr<IOBuf> ZlibCodec::doCompress(const IOBuf* data) {
525 stream.zalloc = nullptr;
526 stream.zfree = nullptr;
527 stream.opaque = nullptr;
529 int rc = deflateInit(&stream, level_);
531 throw std::runtime_error(to<std::string>(
532 "ZlibCodec: deflateInit error: ", rc, ": ", stream.msg));
535 stream.next_in = stream.next_out = nullptr;
536 stream.avail_in = stream.avail_out = 0;
537 stream.total_in = stream.total_out = 0;
539 bool success = false;
542 int rc = deflateEnd(&stream);
543 // If we're here because of an exception, it's okay if some data
545 CHECK(rc == Z_OK || (!success && rc == Z_DATA_ERROR))
546 << rc << ": " << stream.msg;
549 uint64_t uncompressedLength = data->computeChainDataLength();
550 uint64_t maxCompressedLength = deflateBound(&stream, uncompressedLength);
552 // Max 64MiB in one go
553 constexpr uint32_t maxSingleStepLength = uint32_t(64) << 20; // 64MiB
554 constexpr uint32_t defaultBufferLength = uint32_t(4) << 20; // 4MiB
556 auto out = addOutputBuffer(
558 (maxCompressedLength <= maxSingleStepLength ?
559 maxCompressedLength :
560 defaultBufferLength));
562 for (auto& range : *data) {
563 uint64_t remaining = range.size();
564 uint64_t written = 0;
566 uint32_t step = (remaining > maxSingleStepLength ?
567 maxSingleStepLength : remaining);
568 stream.next_in = const_cast<uint8_t*>(range.data() + written);
569 stream.avail_in = step;
573 while (stream.avail_in != 0) {
574 if (stream.avail_out == 0) {
575 out->prependChain(addOutputBuffer(&stream, defaultBufferLength));
578 rc = deflate(&stream, Z_NO_FLUSH);
580 CHECK_EQ(rc, Z_OK) << stream.msg;
586 if (stream.avail_out == 0) {
587 out->prependChain(addOutputBuffer(&stream, defaultBufferLength));
590 rc = deflate(&stream, Z_FINISH);
591 } while (rc == Z_OK);
593 CHECK_EQ(rc, Z_STREAM_END) << stream.msg;
595 out->prev()->trimEnd(stream.avail_out);
597 success = true; // we survived
602 std::unique_ptr<IOBuf> ZlibCodec::doUncompress(const IOBuf* data,
603 uint64_t uncompressedLength) {
605 stream.zalloc = nullptr;
606 stream.zfree = nullptr;
607 stream.opaque = nullptr;
609 int rc = inflateInit(&stream);
611 throw std::runtime_error(to<std::string>(
612 "ZlibCodec: inflateInit error: ", rc, ": ", stream.msg));
615 stream.next_in = stream.next_out = nullptr;
616 stream.avail_in = stream.avail_out = 0;
617 stream.total_in = stream.total_out = 0;
619 bool success = false;
622 int rc = inflateEnd(&stream);
623 // If we're here because of an exception, it's okay if some data
625 CHECK(rc == Z_OK || (!success && rc == Z_DATA_ERROR))
626 << rc << ": " << stream.msg;
629 // Max 64MiB in one go
630 constexpr uint32_t maxSingleStepLength = uint32_t(64) << 20; // 64MiB
631 constexpr uint32_t defaultBufferLength = uint32_t(4) << 20; // 4MiB
633 auto out = addOutputBuffer(
635 ((uncompressedLength != UNKNOWN_UNCOMPRESSED_LENGTH &&
636 uncompressedLength <= maxSingleStepLength) ?
638 defaultBufferLength));
640 bool streamEnd = false;
641 for (auto& range : *data) {
646 stream.next_in = const_cast<uint8_t*>(range.data());
647 stream.avail_in = range.size();
649 while (stream.avail_in != 0) {
651 throw std::runtime_error(to<std::string>(
652 "ZlibCodec: junk after end of data"));
655 streamEnd = doInflate(&stream, out.get(), defaultBufferLength);
660 streamEnd = doInflate(&stream, out.get(), defaultBufferLength);
663 out->prev()->trimEnd(stream.avail_out);
665 if (uncompressedLength != UNKNOWN_UNCOMPRESSED_LENGTH &&
666 uncompressedLength != stream.total_out) {
667 throw std::runtime_error(to<std::string>(
668 "ZlibCodec: invalid uncompressed length"));
671 success = true; // we survived
676 #endif // FOLLY_HAVE_LIBZ
678 #if FOLLY_HAVE_LIBLZMA
683 class LZMA2Codec FOLLY_FINAL : public Codec {
685 static std::unique_ptr<Codec> create(int level, CodecType type);
686 explicit LZMA2Codec(int level, CodecType type);
689 bool doNeedsUncompressedLength() const FOLLY_OVERRIDE;
690 uint64_t doMaxUncompressedLength() const FOLLY_OVERRIDE;
692 bool encodeSize() const { return type() == CodecType::LZMA2_VARINT_SIZE; }
694 std::unique_ptr<IOBuf> doCompress(const IOBuf* data) FOLLY_OVERRIDE;
695 std::unique_ptr<IOBuf> doUncompress(
697 uint64_t uncompressedLength) FOLLY_OVERRIDE;
699 std::unique_ptr<IOBuf> addOutputBuffer(lzma_stream* stream, size_t length);
700 bool doInflate(lzma_stream* stream, IOBuf* head, size_t bufferLength);
705 std::unique_ptr<Codec> LZMA2Codec::create(int level, CodecType type) {
706 return make_unique<LZMA2Codec>(level, type);
709 LZMA2Codec::LZMA2Codec(int level, CodecType type) : Codec(type) {
710 DCHECK(type == CodecType::LZMA2 || type == CodecType::LZMA2_VARINT_SIZE);
712 case COMPRESSION_LEVEL_FASTEST:
715 case COMPRESSION_LEVEL_DEFAULT:
716 level = LZMA_PRESET_DEFAULT;
718 case COMPRESSION_LEVEL_BEST:
722 if (level < 0 || level > 9) {
723 throw std::invalid_argument(to<std::string>(
724 "LZMA2Codec: invalid level: ", level));
729 bool LZMA2Codec::doNeedsUncompressedLength() const {
730 return !encodeSize();
733 uint64_t LZMA2Codec::doMaxUncompressedLength() const {
734 // From lzma/base.h: "Stream is roughly 8 EiB (2^63 bytes)"
735 return uint64_t(1) << 63;
738 std::unique_ptr<IOBuf> LZMA2Codec::addOutputBuffer(
742 CHECK_EQ(stream->avail_out, 0);
744 auto buf = IOBuf::create(length);
747 stream->next_out = buf->writableData();
748 stream->avail_out = buf->length();
753 std::unique_ptr<IOBuf> LZMA2Codec::doCompress(const IOBuf* data) {
755 lzma_stream stream = LZMA_STREAM_INIT;
757 rc = lzma_easy_encoder(&stream, level_, LZMA_CHECK_NONE);
759 throw std::runtime_error(folly::to<std::string>(
760 "LZMA2Codec: lzma_easy_encoder error: ", rc));
763 SCOPE_EXIT { lzma_end(&stream); };
765 uint64_t uncompressedLength = data->computeChainDataLength();
766 uint64_t maxCompressedLength = lzma_stream_buffer_bound(uncompressedLength);
768 // Max 64MiB in one go
769 constexpr uint32_t maxSingleStepLength = uint32_t(64) << 20; // 64MiB
770 constexpr uint32_t defaultBufferLength = uint32_t(4) << 20; // 4MiB
772 auto out = addOutputBuffer(
774 (maxCompressedLength <= maxSingleStepLength ?
775 maxCompressedLength :
776 defaultBufferLength));
779 auto size = IOBuf::createCombined(kMaxVarintLength64);
780 encodeVarintToIOBuf(uncompressedLength, size.get());
781 size->appendChain(std::move(out));
782 out = std::move(size);
785 for (auto& range : *data) {
790 stream.next_in = const_cast<uint8_t*>(range.data());
791 stream.avail_in = range.size();
793 while (stream.avail_in != 0) {
794 if (stream.avail_out == 0) {
795 out->prependChain(addOutputBuffer(&stream, defaultBufferLength));
798 rc = lzma_code(&stream, LZMA_RUN);
801 throw std::runtime_error(folly::to<std::string>(
802 "LZMA2Codec: lzma_code error: ", rc));
808 if (stream.avail_out == 0) {
809 out->prependChain(addOutputBuffer(&stream, defaultBufferLength));
812 rc = lzma_code(&stream, LZMA_FINISH);
813 } while (rc == LZMA_OK);
815 if (rc != LZMA_STREAM_END) {
816 throw std::runtime_error(folly::to<std::string>(
817 "LZMA2Codec: lzma_code ended with error: ", rc));
820 out->prev()->trimEnd(stream.avail_out);
825 bool LZMA2Codec::doInflate(lzma_stream* stream,
827 size_t bufferLength) {
828 if (stream->avail_out == 0) {
829 head->prependChain(addOutputBuffer(stream, bufferLength));
832 lzma_ret rc = lzma_code(stream, LZMA_RUN);
837 case LZMA_STREAM_END:
840 throw std::runtime_error(to<std::string>(
841 "LZMA2Codec: lzma_code error: ", rc));
847 std::unique_ptr<IOBuf> LZMA2Codec::doUncompress(const IOBuf* data,
848 uint64_t uncompressedLength) {
850 lzma_stream stream = LZMA_STREAM_INIT;
852 rc = lzma_auto_decoder(&stream, std::numeric_limits<uint64_t>::max(), 0);
854 throw std::runtime_error(folly::to<std::string>(
855 "LZMA2Codec: lzma_auto_decoder error: ", rc));
858 SCOPE_EXIT { lzma_end(&stream); };
860 // Max 64MiB in one go
861 constexpr uint32_t maxSingleStepLength = uint32_t(64) << 20; // 64MiB
862 constexpr uint32_t defaultBufferLength = uint32_t(4) << 20; // 4MiB
864 folly::io::Cursor cursor(data);
865 uint64_t actualUncompressedLength;
867 actualUncompressedLength = decodeVarintFromCursor(cursor);
868 if (uncompressedLength != UNKNOWN_UNCOMPRESSED_LENGTH &&
869 uncompressedLength != actualUncompressedLength) {
870 throw std::runtime_error("LZMA2Codec: invalid uncompressed length");
873 actualUncompressedLength = uncompressedLength;
874 DCHECK_NE(actualUncompressedLength, UNKNOWN_UNCOMPRESSED_LENGTH);
877 auto out = addOutputBuffer(
879 (actualUncompressedLength <= maxSingleStepLength ?
880 actualUncompressedLength :
881 defaultBufferLength));
883 bool streamEnd = false;
884 auto buf = cursor.peek();
885 while (buf.second != 0) {
886 stream.next_in = const_cast<uint8_t*>(buf.first);
887 stream.avail_in = buf.second;
889 while (stream.avail_in != 0) {
891 throw std::runtime_error(to<std::string>(
892 "LZMA2Codec: junk after end of data"));
895 streamEnd = doInflate(&stream, out.get(), defaultBufferLength);
898 cursor.skip(buf.second);
903 streamEnd = doInflate(&stream, out.get(), defaultBufferLength);
906 out->prev()->trimEnd(stream.avail_out);
908 if (actualUncompressedLength != stream.total_out) {
909 throw std::runtime_error(to<std::string>(
910 "LZMA2Codec: invalid uncompressed length"));
916 #endif // FOLLY_HAVE_LIBLZMA
918 typedef std::unique_ptr<Codec> (*CodecFactory)(int, CodecType);
920 CodecFactory gCodecFactories[
921 static_cast<size_t>(CodecType::NUM_CODEC_TYPES)] = {
922 nullptr, // USER_DEFINED
923 NoCompressionCodec::create,
925 #if FOLLY_HAVE_LIBLZ4
931 #if FOLLY_HAVE_LIBSNAPPY
943 #if FOLLY_HAVE_LIBLZ4
949 #if FOLLY_HAVE_LIBLZMA
960 std::unique_ptr<Codec> getCodec(CodecType type, int level) {
961 size_t idx = static_cast<size_t>(type);
962 if (idx >= static_cast<size_t>(CodecType::NUM_CODEC_TYPES)) {
963 throw std::invalid_argument(to<std::string>(
964 "Compression type ", idx, " not supported"));
966 auto factory = gCodecFactories[idx];
968 throw std::invalid_argument(to<std::string>(
969 "Compression type ", idx, " not supported"));
971 auto codec = (*factory)(level, type);
972 DCHECK_EQ(static_cast<size_t>(codec->type()), idx);