From e6cef616f1f9882119e1a7caf84debb247186c6b Mon Sep 17 00:00:00 2001 From: David Anderson Date: Thu, 26 Jan 2023 20:49:11 -0800 Subject: [PATCH] liblp: Add a helper class for building sparse-compatible super image layouts. This class helps export a list of regions comprising a super partition, and what the contents of those regions should be. It is very similar to ImageBuilder, except that it does not require actual partition image files, nor does it actually write an image file to disk. The goal is to support building an in-memory super image that uses as little memory and backing storage as possible. For example, fastboot can use this to upload a super image without having to read and write gigabytes of unnecessary data. Since the goal is to optimize fastboot, we've taken some shortcuts here. Retrofit devices and other edge-casey behavior are safely rejected. We also don't rely on libsparse behavior here, and attempt to make the translation to sparse records as trivial as possible, by explicitly declaring where fill/dontcare gaps are, and only exporting 4KB aligned regions. Hopefully this will allow the code to be portable to non-fastboot consumers. Bug: 266982466 Test: liblp_test Change-Id: I1b41d233bc4512c4b62e19603e8e77bc5867cfab --- fs_mgr/liblp/Android.bp | 35 ++- .../include/liblp/super_layout_builder.h | 104 ++++++++ fs_mgr/liblp/liblp_test.xml | 26 -- fs_mgr/liblp/super_layout_builder.cpp | 241 ++++++++++++++++++ fs_mgr/liblp/super_layout_builder_test.cpp | 141 ++++++++++ 5 files changed, 507 insertions(+), 40 deletions(-) create mode 100644 fs_mgr/liblp/include/liblp/super_layout_builder.h delete mode 100644 fs_mgr/liblp/liblp_test.xml create mode 100644 fs_mgr/liblp/super_layout_builder.cpp create mode 100644 fs_mgr/liblp/super_layout_builder_test.cpp diff --git a/fs_mgr/liblp/Android.bp b/fs_mgr/liblp/Android.bp index 4b81c2c07..996ffd759 100644 --- a/fs_mgr/liblp/Android.bp +++ b/fs_mgr/liblp/Android.bp @@ -39,6 +39,7 @@ cc_library { ], srcs: [ "builder.cpp", + "super_layout_builder.cpp", "images.cpp", "partition_opener.cpp", "property_fetcher.cpp", @@ -62,17 +63,6 @@ cc_library { export_include_dirs: ["include"], } -filegroup { - name: "liblp_test_srcs", - srcs: [ - "builder_test.cpp", - "device_test.cpp", - "io_test.cpp", - "test_partition_opener.cpp", - "utility_test.cpp", - ], -} - cc_defaults { name: "liblp_test_defaults", defaults: ["fs_mgr_defaults"], @@ -82,22 +72,39 @@ cc_defaults { static_libs: [ "libcutils", "libgmock", - "libfs_mgr", "liblp", "libcrypto_static", ] + liblp_lib_deps, header_libs: [ "libstorage_literals_headers", ], + target: { + android: { + srcs: [ + "device_test.cpp", + "io_test.cpp", + ], + static_libs: [ + "libfs_mgr", + ], + } + }, stl: "libc++_static", - srcs: [":liblp_test_srcs"], + srcs: [ + "builder_test.cpp", + "super_layout_builder_test.cpp", + "test_partition_opener.cpp", + "utility_test.cpp", + ], } cc_test { name: "liblp_test", defaults: ["liblp_test_defaults"], - test_config: "liblp_test.xml", test_suites: ["device-tests"], + auto_gen_config: true, + require_root: true, + host_supported: true } cc_test { diff --git a/fs_mgr/liblp/include/liblp/super_layout_builder.h b/fs_mgr/liblp/include/liblp/super_layout_builder.h new file mode 100644 index 000000000..d92085537 --- /dev/null +++ b/fs_mgr/liblp/include/liblp/super_layout_builder.h @@ -0,0 +1,104 @@ +// +// Copyright (C) 2023 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +#pragma once + +#include + +#include +#include +#include +#include +#include + +#include +#include + +namespace android { +namespace fs_mgr { + +struct SuperImageExtent { + enum class Type { INVALID, DATA, PARTITION, ZERO, DONTCARE }; + + SuperImageExtent(const SuperImageExtent& other) = default; + SuperImageExtent(SuperImageExtent&& other) = default; + SuperImageExtent(uint64_t offset, uint64_t size, Type type) + : offset(offset), size(size), type(type) {} + + SuperImageExtent(uint64_t offset, std::shared_ptr blob) + : SuperImageExtent(offset, blob->size(), Type::DATA) { + this->blob = blob; + } + + SuperImageExtent(uint64_t offset, uint64_t size, const std::string& image_name, + uint64_t image_offset) + : SuperImageExtent(offset, size, Type::PARTITION) { + this->image_name = image_name; + this->image_offset = image_offset; + } + + SuperImageExtent& operator=(const SuperImageExtent& other) = default; + SuperImageExtent& operator=(SuperImageExtent&& other) = default; + + bool operator<(const SuperImageExtent& other) const { return offset < other.offset; } + bool operator==(const SuperImageExtent& other) const; + + // Location, size, and type of the extent. + uint64_t offset = 0; + uint64_t size = 0; + Type type = Type::INVALID; + + // If type == DATA, this contains the bytes to write. + std::shared_ptr blob; + // If type == PARTITION, this contains the partition image name and + // offset within that file. + std::string image_name; + uint64_t image_offset = 0; +}; + +// The SuperLayoutBuilder allows building a sparse view of a super image. This +// is useful for efficient flashing, eg to bypass fastbootd and directly flash +// super without physically building and storing the image. +class SuperLayoutBuilder final { + public: + // Open a super_empty.img, return false on failure. This must be called to + // initialize the tool. If it returns false, either the image failed to + // parse, or the tool is not compatible with how the device is configured + // (in which case fastbootd should be preferred). + [[nodiscard]] bool Open(android::base::borrowed_fd fd); + [[nodiscard]] bool Open(const void* data, size_t bytes); + [[nodiscard]] bool Open(const LpMetadata& metadata); + + // Add a partition's image and size to the work list. If false is returned, + // there was either a duplicate partition or not enough space in super. + bool AddPartition(const std::string& partition_name, const std::string& image_name, + uint64_t partition_size); + + // Return the list of extents describing the super image. If this list is + // empty, then there was an unrecoverable error in building the list. + std::vector GetImageLayout(); + + // Return the current metadata. + std::unique_ptr Export() const { return builder_->Export(); } + + private: + std::unique_ptr builder_; + std::unordered_map image_map_; +}; + +std::ostream& operator<<(std::ostream& stream, const SuperImageExtent& extent); + +} // namespace fs_mgr +} // namespace android diff --git a/fs_mgr/liblp/liblp_test.xml b/fs_mgr/liblp/liblp_test.xml deleted file mode 100644 index 98414b109..000000000 --- a/fs_mgr/liblp/liblp_test.xml +++ /dev/null @@ -1,26 +0,0 @@ - - - - - - - - - diff --git a/fs_mgr/liblp/super_layout_builder.cpp b/fs_mgr/liblp/super_layout_builder.cpp new file mode 100644 index 000000000..37f28e124 --- /dev/null +++ b/fs_mgr/liblp/super_layout_builder.cpp @@ -0,0 +1,241 @@ +// +// Copyright (C) 2023 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +#include + +#include + +#include "images.h" +#include "utility.h" +#include "writer.h" + +using android::base::borrowed_fd; +using android::base::unique_fd; + +namespace android { +namespace fs_mgr { + +bool SuperLayoutBuilder::Open(borrowed_fd fd) { + auto metadata = ReadFromImageFile(fd.get()); + if (!metadata) { + return false; + } + return Open(*metadata.get()); +} + +bool SuperLayoutBuilder::Open(const void* data, size_t size) { + auto metadata = ReadFromImageBlob(data, size); + if (!metadata) { + return false; + } + return Open(*metadata.get()); +} + +bool SuperLayoutBuilder::Open(const LpMetadata& metadata) { + for (const auto& partition : metadata.partitions) { + if (partition.attributes & LP_PARTITION_ATTR_SLOT_SUFFIXED) { + // Retrofit devices are not supported. + return false; + } + if (!(partition.attributes & LP_PARTITION_ATTR_READONLY)) { + // Writable partitions are not supported. + return false; + } + } + if (!metadata.extents.empty()) { + // Partitions that already have extents are not supported (should + // never be true of super_empty.img). + return false; + } + if (metadata.block_devices.size() != 1) { + // Only one "super" is supported. + return false; + } + + builder_ = MetadataBuilder::New(metadata); + return !!builder_; +} + +bool SuperLayoutBuilder::AddPartition(const std::string& partition_name, + const std::string& image_name, uint64_t partition_size) { + auto p = builder_->FindPartition(partition_name); + if (!p) { + return false; + } + if (!builder_->ResizePartition(p, partition_size)) { + return false; + } + image_map_.emplace(partition_name, image_name); + return true; +} + +// Fill the space between each extent, if any, with either a fill or dontcare +// extent. The caller constructs a sample extent to re-use. +static bool AddGapExtents(std::vector* extents, SuperImageExtent::Type gap_type) { + std::vector old = std::move(*extents); + std::sort(old.begin(), old.end()); + + *extents = {}; + + uint64_t current_offset = 0; + for (const auto& extent : old) { + // Check for overlapping extents - this would be a serious error. + if (current_offset > extent.offset) { + LOG(INFO) << "Overlapping extents detected; cannot layout temporary super image"; + return false; + } + + if (extent.offset != current_offset) { + uint64_t gap_size = extent.offset - current_offset; + extents->emplace_back(current_offset, gap_size, gap_type); + current_offset = extent.offset; + } + + extents->emplace_back(extent); + current_offset += extent.size; + } + return true; +} + +std::vector SuperLayoutBuilder::GetImageLayout() { + auto metadata = builder_->Export(); + if (!metadata) { + return {}; + } + + std::vector extents; + + // Write the primary and backup copies of geometry. + std::string geometry_bytes = SerializeGeometry(metadata->geometry); + auto blob = std::make_shared(std::move(geometry_bytes)); + + extents.emplace_back(0, GetPrimaryGeometryOffset(), SuperImageExtent::Type::ZERO); + extents.emplace_back(GetPrimaryGeometryOffset(), blob); + extents.emplace_back(GetBackupGeometryOffset(), blob); + + // Write the primary and backup copies of each metadata slot. When flashing, + // all metadata copies are the same, even for different slots. + std::string metadata_bytes = SerializeMetadata(*metadata.get()); + + // Align metadata size to 4KB. This makes the layout easily compatible with + // libsparse. + static constexpr size_t kSparseAlignment = 4096; + size_t metadata_aligned_bytes; + if (!AlignTo(metadata_bytes.size(), kSparseAlignment, &metadata_aligned_bytes)) { + LOG(ERROR) << "Unable to align metadata size " << metadata_bytes.size() << " to " + << kSparseAlignment; + return {}; + } + metadata_bytes.resize(metadata_aligned_bytes, '\0'); + + // However, alignment can cause larger-than-supported metadata blocks. Fall + // back to fastbootd/update-super. + if (metadata_bytes.size() > metadata->geometry.metadata_max_size) { + LOG(VERBOSE) << "Aligned metadata size " << metadata_bytes.size() + << " is larger than maximum metadata size " + << metadata->geometry.metadata_max_size; + return {}; + } + + blob = std::make_shared(std::move(metadata_bytes)); + for (uint32_t i = 0; i < metadata->geometry.metadata_slot_count; i++) { + int64_t metadata_primary = GetPrimaryMetadataOffset(metadata->geometry, i); + int64_t metadata_backup = GetBackupMetadataOffset(metadata->geometry, i); + extents.emplace_back(metadata_primary, blob); + extents.emplace_back(metadata_backup, blob); + } + + // Add extents for each partition. + for (const auto& partition : metadata->partitions) { + auto partition_name = GetPartitionName(partition); + auto image_name_iter = image_map_.find(partition_name); + if (image_name_iter == image_map_.end()) { + if (partition.num_extents != 0) { + LOG(ERROR) << "Partition " << partition_name + << " has extents but no image filename"; + return {}; + } + continue; + } + const auto& image_name = image_name_iter->second; + + uint64_t image_offset = 0; + for (uint32_t i = 0; i < partition.num_extents; i++) { + const auto& e = metadata->extents[partition.first_extent_index + i]; + + if (e.target_type != LP_TARGET_TYPE_LINEAR) { + // Any type other than LINEAR isn't understood here. We don't even + // bother with ZERO, which is never generated. + LOG(INFO) << "Unknown extent type from liblp: " << e.target_type; + return {}; + } + + size_t size = e.num_sectors * LP_SECTOR_SIZE; + uint64_t super_offset = e.target_data * LP_SECTOR_SIZE; + extents.emplace_back(super_offset, size, image_name, image_offset); + + image_offset += size; + } + } + + if (!AddGapExtents(&extents, SuperImageExtent::Type::DONTCARE)) { + return {}; + } + return extents; +} + +bool SuperImageExtent::operator==(const SuperImageExtent& other) const { + if (offset != other.offset) { + return false; + } + if (size != other.size) { + return false; + } + if (type != other.type) { + return false; + } + switch (type) { + case Type::DATA: + return *blob == *other.blob; + case Type::PARTITION: + return image_name == other.image_name && image_offset == other.image_offset; + default: + return true; + } +} + +std::ostream& operator<<(std::ostream& stream, const SuperImageExtent& extent) { + stream << "extent:" << extent.offset << ":" << extent.size << ":"; + switch (extent.type) { + case SuperImageExtent::Type::DATA: + stream << "data"; + break; + case SuperImageExtent::Type::PARTITION: + stream << "partition:" << extent.image_name << ":" << extent.image_offset; + break; + case SuperImageExtent::Type::ZERO: + stream << "zero"; + break; + case SuperImageExtent::Type::DONTCARE: + stream << "dontcare"; + break; + default: + stream << "invalid"; + } + return stream; +} + +} // namespace fs_mgr +} // namespace android diff --git a/fs_mgr/liblp/super_layout_builder_test.cpp b/fs_mgr/liblp/super_layout_builder_test.cpp new file mode 100644 index 000000000..714b6b439 --- /dev/null +++ b/fs_mgr/liblp/super_layout_builder_test.cpp @@ -0,0 +1,141 @@ +// +// Copyright (C) 2023 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +#include +#include +#include +#include +#include + +#include "images.h" +#include "writer.h" + +using namespace android::fs_mgr; +using namespace android::storage_literals; + +TEST(SuperImageTool, Layout) { + auto builder = MetadataBuilder::New(4_MiB, 8_KiB, 2); + ASSERT_NE(builder, nullptr); + + Partition* p = builder->AddPartition("system_a", LP_PARTITION_ATTR_READONLY); + ASSERT_NE(p, nullptr); + + auto metadata = builder->Export(); + ASSERT_NE(metadata, nullptr); + + SuperLayoutBuilder tool; + ASSERT_TRUE(tool.Open(*metadata.get())); + ASSERT_TRUE(tool.AddPartition("system_a", "system.img", 16_KiB)); + + // Get a copy of the metadata we'd expect if flashing. + ASSERT_TRUE(builder->ResizePartition(p, 16_KiB)); + metadata = builder->Export(); + ASSERT_NE(metadata, nullptr); + + auto geometry_blob = std::make_shared(SerializeGeometry(metadata->geometry)); + auto metadata_blob = std::make_shared(SerializeMetadata(*metadata.get())); + metadata_blob->resize(4_KiB, '\0'); + + auto extents = tool.GetImageLayout(); + ASSERT_EQ(extents.size(), 12); + EXPECT_EQ(extents[0], SuperImageExtent(0, 4096, SuperImageExtent::Type::ZERO)); + EXPECT_EQ(extents[1], SuperImageExtent(4096, geometry_blob)); + EXPECT_EQ(extents[2], SuperImageExtent(8192, geometry_blob)); + EXPECT_EQ(extents[3], SuperImageExtent(12288, metadata_blob)); + EXPECT_EQ(extents[4], SuperImageExtent(16384, 4096, SuperImageExtent::Type::DONTCARE)); + EXPECT_EQ(extents[5], SuperImageExtent(20480, metadata_blob)); + EXPECT_EQ(extents[6], SuperImageExtent(24576, 4096, SuperImageExtent::Type::DONTCARE)); + EXPECT_EQ(extents[7], SuperImageExtent(28672, metadata_blob)); + EXPECT_EQ(extents[8], SuperImageExtent(32768, 4096, SuperImageExtent::Type::DONTCARE)); + EXPECT_EQ(extents[9], SuperImageExtent(36864, metadata_blob)); + EXPECT_EQ(extents[10], SuperImageExtent(40960, 4096, SuperImageExtent::Type::DONTCARE)); + EXPECT_EQ(extents[11], SuperImageExtent(45056, 16384, "system.img", 0)); +} + +TEST(SuperImageTool, NoWritablePartitions) { + auto builder = MetadataBuilder::New(4_MiB, 8_KiB, 2); + ASSERT_NE(builder, nullptr); + + Partition* p = builder->AddPartition("system_a", 0); + ASSERT_NE(p, nullptr); + + auto metadata = builder->Export(); + ASSERT_NE(metadata, nullptr); + + SuperLayoutBuilder tool; + ASSERT_FALSE(tool.Open(*metadata.get())); +} + +TEST(SuperImageTool, NoRetrofit) { + auto builder = MetadataBuilder::New(4_MiB, 8_KiB, 2); + ASSERT_NE(builder, nullptr); + + Partition* p = builder->AddPartition("system_a", LP_PARTITION_ATTR_READONLY); + ASSERT_NE(p, nullptr); + + auto metadata = builder->Export(); + ASSERT_NE(metadata, nullptr); + + // Add an extra block device. + metadata->block_devices.emplace_back(metadata->block_devices[0]); + + SuperLayoutBuilder tool; + ASSERT_FALSE(tool.Open(*metadata.get())); +} + +TEST(SuperImageTool, NoRetrofit2) { + auto builder = MetadataBuilder::New(4_MiB, 8_KiB, 2); + ASSERT_NE(builder, nullptr); + + Partition* p = builder->AddPartition( + "system_a", LP_PARTITION_ATTR_READONLY | LP_PARTITION_ATTR_SLOT_SUFFIXED); + ASSERT_NE(p, nullptr); + + auto metadata = builder->Export(); + ASSERT_NE(metadata, nullptr); + + SuperLayoutBuilder tool; + ASSERT_FALSE(tool.Open(*metadata.get())); +} + +TEST(SuperImageTool, NoFixedPartitions) { + auto builder = MetadataBuilder::New(4_MiB, 8_KiB, 2); + ASSERT_NE(builder, nullptr); + + Partition* p = builder->AddPartition("system_a", LP_PARTITION_ATTR_READONLY); + ASSERT_NE(p, nullptr); + ASSERT_TRUE(builder->ResizePartition(p, 4_KiB)); + + auto metadata = builder->Export(); + ASSERT_NE(metadata, nullptr); + + SuperLayoutBuilder tool; + ASSERT_FALSE(tool.Open(*metadata.get())); +} + +TEST(SuperImageTool, LargeAlignedMetadata) { + auto builder = MetadataBuilder::New(4_MiB, 512, 2); + ASSERT_NE(builder, nullptr); + + auto metadata = builder->Export(); + ASSERT_NE(metadata, nullptr); + + SuperLayoutBuilder tool; + ASSERT_TRUE(tool.Open(*metadata.get())); + + auto extents = tool.GetImageLayout(); + ASSERT_TRUE(extents.empty()); +}