From a652877bd6a142cd98f704f73b1c3fd472ca57ab Mon Sep 17 00:00:00 2001 From: David Anderson Date: Fri, 4 Sep 2020 15:47:04 -0700 Subject: [PATCH] libsnapshot: Add support for brotli compression. Bug: 162274240 Test: cow_api_test Change-Id: I0b0ceec3c3041a6aea4b1e6c4d01ed0a8860d7e8 --- fs_mgr/libsnapshot/Android.bp | 10 ++-- fs_mgr/libsnapshot/cow_api_test.cpp | 12 +++-- fs_mgr/libsnapshot/cow_decompress.cpp | 53 +++++++++++++++++++ fs_mgr/libsnapshot/cow_decompress.h | 1 + fs_mgr/libsnapshot/cow_reader.cpp | 3 ++ fs_mgr/libsnapshot/cow_writer.cpp | 23 ++++++++ .../include/libsnapshot/cow_format.h | 1 + 7 files changed, 96 insertions(+), 7 deletions(-) diff --git a/fs_mgr/libsnapshot/Android.bp b/fs_mgr/libsnapshot/Android.bp index bdf1da682..95fbab8ba 100644 --- a/fs_mgr/libsnapshot/Android.bp +++ b/fs_mgr/libsnapshot/Android.bp @@ -152,6 +152,7 @@ cc_library_static { "liblog", ], static_libs: [ + "libbrotli", "libz", ], ramdisk_available: true, @@ -362,10 +363,11 @@ cc_defaults { static_libs: [ "libbase", + "libbrotli", "liblog", "libdm", - "libz", - "libsnapshot_cow", + "libz", + "libsnapshot_cow", ], } @@ -403,6 +405,7 @@ cc_test { "libz", ], static_libs: [ + "libbrotli", "libgtest", "libsnapshot_cow", ], @@ -494,11 +497,12 @@ cc_test { shared_libs: [ "libbase", "liblog", - "libz", ], static_libs: [ + "libbrotli", "libgtest", "libsnapshot_cow", + "libz", ], header_libs: [ "libstorage_literals_headers", diff --git a/fs_mgr/libsnapshot/cow_api_test.cpp b/fs_mgr/libsnapshot/cow_api_test.cpp index d98fe59c1..ec206ad9a 100644 --- a/fs_mgr/libsnapshot/cow_api_test.cpp +++ b/fs_mgr/libsnapshot/cow_api_test.cpp @@ -30,12 +30,12 @@ namespace snapshot { class CowTest : public ::testing::Test { protected: - void SetUp() override { + virtual void SetUp() override { cow_ = std::make_unique(); ASSERT_GE(cow_->fd, 0) << strerror(errno); } - void TearDown() override { cow_ = nullptr; } + virtual void TearDown() override { cow_ = nullptr; } std::unique_ptr cow_; }; @@ -211,9 +211,11 @@ class HorribleStringSink : public StringSink { void* GetBuffer(size_t, size_t* actual) override { return StringSink::GetBuffer(1, actual); } }; -TEST_F(CowTest, HorribleSink) { +class CompressionTest : public CowTest, public testing::WithParamInterface {}; + +TEST_P(CompressionTest, HorribleSink) { CowOptions options; - options.compression = "gz"; + options.compression = GetParam(); CowWriter writer(options); ASSERT_TRUE(writer.Initialize(cow_->fd)); @@ -239,6 +241,8 @@ TEST_F(CowTest, HorribleSink) { ASSERT_EQ(sink.stream(), data); } +INSTANTIATE_TEST_SUITE_P(CowApi, CompressionTest, testing::Values("none", "gz", "brotli")); + TEST_F(CowTest, GetSize) { CowOptions options; CowWriter writer(options); diff --git a/fs_mgr/libsnapshot/cow_decompress.cpp b/fs_mgr/libsnapshot/cow_decompress.cpp index f480b85df..faceafe17 100644 --- a/fs_mgr/libsnapshot/cow_decompress.cpp +++ b/fs_mgr/libsnapshot/cow_decompress.cpp @@ -19,6 +19,7 @@ #include #include +#include #include namespace android { @@ -207,5 +208,57 @@ std::unique_ptr IDecompressor::Gz() { return std::unique_ptr(new GzDecompressor()); } +class BrotliDecompressor final : public StreamDecompressor { + public: + ~BrotliDecompressor(); + + bool Init() override; + bool DecompressInput(const uint8_t* data, size_t length) override; + bool Done() override { return BrotliDecoderIsFinished(decoder_); } + + private: + BrotliDecoderState* decoder_ = nullptr; +}; + +bool BrotliDecompressor::Init() { + decoder_ = BrotliDecoderCreateInstance(nullptr, nullptr, nullptr); + return true; +} + +BrotliDecompressor::~BrotliDecompressor() { + if (decoder_) { + BrotliDecoderDestroyInstance(decoder_); + } +} + +bool BrotliDecompressor::DecompressInput(const uint8_t* data, size_t length) { + size_t available_in = length; + const uint8_t* next_in = data; + + bool needs_more_output = false; + while (available_in || needs_more_output) { + if (!output_buffer_remaining_ && !GetFreshBuffer()) { + return false; + } + + auto output_buffer = output_buffer_; + auto r = BrotliDecoderDecompressStream(decoder_, &available_in, &next_in, + &output_buffer_remaining_, &output_buffer_, nullptr); + if (r == BROTLI_DECODER_RESULT_ERROR) { + LOG(ERROR) << "brotli decode failed"; + return false; + } + if (!sink_->ReturnData(output_buffer, output_buffer_ - output_buffer)) { + return false; + } + needs_more_output = (r == BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT); + } + return true; +} + +std::unique_ptr IDecompressor::Brotli() { + return std::unique_ptr(new BrotliDecompressor()); +} + } // namespace snapshot } // namespace android diff --git a/fs_mgr/libsnapshot/cow_decompress.h b/fs_mgr/libsnapshot/cow_decompress.h index 1c8c40d91..f4852561f 100644 --- a/fs_mgr/libsnapshot/cow_decompress.h +++ b/fs_mgr/libsnapshot/cow_decompress.h @@ -40,6 +40,7 @@ class IDecompressor { // Factory methods for decompression methods. static std::unique_ptr Uncompressed(); static std::unique_ptr Gz(); + static std::unique_ptr Brotli(); // |output_bytes| is the expected total number of bytes to sink. virtual bool Decompress(size_t output_bytes) = 0; diff --git a/fs_mgr/libsnapshot/cow_reader.cpp b/fs_mgr/libsnapshot/cow_reader.cpp index 1aea3a92d..720ee3b45 100644 --- a/fs_mgr/libsnapshot/cow_reader.cpp +++ b/fs_mgr/libsnapshot/cow_reader.cpp @@ -233,6 +233,9 @@ bool CowReader::ReadData(const CowOperation& op, IByteSink* sink) { case kCowCompressGz: decompressor = IDecompressor::Gz(); break; + case kCowCompressBrotli: + decompressor = IDecompressor::Brotli(); + break; default: LOG(ERROR) << "Unknown compression type: " << op.compression; return false; diff --git a/fs_mgr/libsnapshot/cow_writer.cpp b/fs_mgr/libsnapshot/cow_writer.cpp index 76238c262..f05f9ba4b 100644 --- a/fs_mgr/libsnapshot/cow_writer.cpp +++ b/fs_mgr/libsnapshot/cow_writer.cpp @@ -22,6 +22,7 @@ #include #include #include +#include #include #include @@ -63,6 +64,10 @@ bool CowWriter::Initialize(android::base::borrowed_fd fd) { if (options_.compression == "gz") { compression_ = kCowCompressGz; + } else if (options_.compression == "brotli") { + compression_ = kCowCompressBrotli; + } else if (options_.compression == "none") { + compression_ = kCowCompressNone; } else if (!options_.compression.empty()) { LOG(ERROR) << "unrecognized compression: " << options_.compression; return false; @@ -171,6 +176,24 @@ std::basic_string CowWriter::Compress(const void* data, size_t length) } return std::basic_string(buffer.get(), dest_len); } + case kCowCompressBrotli: { + auto bound = BrotliEncoderMaxCompressedSize(length); + if (!bound) { + LOG(ERROR) << "BrotliEncoderMaxCompressedSize returned 0"; + return {}; + } + auto buffer = std::make_unique(bound); + + size_t encoded_size = bound; + auto rv = BrotliEncoderCompress( + BROTLI_DEFAULT_QUALITY, BROTLI_DEFAULT_WINDOW, BROTLI_DEFAULT_MODE, length, + reinterpret_cast(data), &encoded_size, buffer.get()); + if (!rv) { + LOG(ERROR) << "BrotliEncoderCompress failed"; + return {}; + } + return std::basic_string(buffer.get(), encoded_size); + } default: LOG(ERROR) << "unhandled compression type: " << compression_; break; diff --git a/fs_mgr/libsnapshot/include/libsnapshot/cow_format.h b/fs_mgr/libsnapshot/include/libsnapshot/cow_format.h index 6d500e7e5..0adce4867 100644 --- a/fs_mgr/libsnapshot/include/libsnapshot/cow_format.h +++ b/fs_mgr/libsnapshot/include/libsnapshot/cow_format.h @@ -98,6 +98,7 @@ static constexpr uint8_t kCowZeroOp = 3; static constexpr uint8_t kCowCompressNone = 0; static constexpr uint8_t kCowCompressGz = 1; +static constexpr uint8_t kCowCompressBrotli = 2; } // namespace snapshot } // namespace android