pa-fazil
/
android_system_core
mirror of https://github.com/fazilsheik96/android_system_core.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

liblp: Add a helper class for building sparse-compatible super image layouts. 

This class helps export a list of regions comprising a super partition,
and what the contents of those regions should be. It is very similar to
ImageBuilder, except that it does not require actual partition image
files, nor does it actually write an image file to disk.

The goal is to support building an in-memory super image that uses as
little memory and backing storage as possible. For example, fastboot can
use this to upload a super image without having to read and write
gigabytes of unnecessary data.

Since the goal is to optimize fastboot, we've taken some shortcuts here.
Retrofit devices and other edge-casey behavior are safely rejected. We
also don't rely on libsparse behavior here, and attempt to make the
translation to sparse records as trivial as possible, by explicitly
declaring where fill/dontcare gaps are, and only exporting 4KB aligned
regions. Hopefully this will allow the code to be portable to
non-fastboot consumers.

Bug: 266982466
Test: liblp_test
Change-Id: I1b41d233bc4512c4b62e19603e8e77bc5867cfab
This commit is contained in:
David Anderson 2023-01-26 20:49:11 -08:00
parent d34157e26e
commit e6cef616f1
5 changed files with 507 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
fs_mgr/liblp/Android.bp
View File

@ -39,6 +39,7 @@ cc_library {
],
srcs: [
"builder.cpp",
"super_layout_builder.cpp",
"images.cpp",
"partition_opener.cpp",
"property_fetcher.cpp",
@ -62,17 +63,6 @@ cc_library {
export_include_dirs: ["include"],
}
filegroup {
name: "liblp_test_srcs",
srcs: [
"builder_test.cpp",
"device_test.cpp",
"io_test.cpp",
"test_partition_opener.cpp",
"utility_test.cpp",
],
}
cc_defaults {
name: "liblp_test_defaults",
defaults: ["fs_mgr_defaults"],
@ -82,22 +72,39 @@ cc_defaults {
static_libs: [
"libcutils",
"libgmock",
"libfs_mgr",
"liblp",
"libcrypto_static",
] + liblp_lib_deps,
header_libs: [
"libstorage_literals_headers",
],
target: {
android: {
srcs: [
"device_test.cpp",
"io_test.cpp",
],
static_libs: [
"libfs_mgr",
],
}
},
stl: "libc++_static",
srcs: [":liblp_test_srcs"],
srcs: [
"builder_test.cpp",
"super_layout_builder_test.cpp",
"test_partition_opener.cpp",
"utility_test.cpp",
],
}
cc_test {
name: "liblp_test",
defaults: ["liblp_test_defaults"],
test_config: "liblp_test.xml",
test_suites: ["device-tests"],
auto_gen_config: true,
require_root: true,
host_supported: true
}
cc_test {

104
fs_mgr/liblp/include/liblp/super_layout_builder.h Normal file
View File

@ -0,0 +1,104 @@
//
// Copyright (C) 2023 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
#pragma once
#include <stdint.h>
#include <memory>
#include <ostream>
#include <string>
#include <unordered_map>
#include <utility>
#include <android-base/unique_fd.h>
#include <liblp/builder.h>
namespace android {
namespace fs_mgr {
struct SuperImageExtent {
enum class Type { INVALID, DATA, PARTITION, ZERO, DONTCARE };
SuperImageExtent(const SuperImageExtent& other) = default;
SuperImageExtent(SuperImageExtent&& other) = default;
SuperImageExtent(uint64_t offset, uint64_t size, Type type)
: offset(offset), size(size), type(type) {}
SuperImageExtent(uint64_t offset, std::shared_ptr<std::string> blob)
: SuperImageExtent(offset, blob->size(), Type::DATA) {
this->blob = blob;
}
SuperImageExtent(uint64_t offset, uint64_t size, const std::string& image_name,
uint64_t image_offset)
: SuperImageExtent(offset, size, Type::PARTITION) {
this->image_name = image_name;
this->image_offset = image_offset;
}
SuperImageExtent& operator=(const SuperImageExtent& other) = default;
SuperImageExtent& operator=(SuperImageExtent&& other) = default;
bool operator<(const SuperImageExtent& other) const { return offset < other.offset; }
bool operator==(const SuperImageExtent& other) const;
// Location, size, and type of the extent.
uint64_t offset = 0;
uint64_t size = 0;
Type type = Type::INVALID;
// If type == DATA, this contains the bytes to write.
std::shared_ptr<std::string> blob;
// If type == PARTITION, this contains the partition image name and
// offset within that file.
std::string image_name;
uint64_t image_offset = 0;
};
// The SuperLayoutBuilder allows building a sparse view of a super image. This
// is useful for efficient flashing, eg to bypass fastbootd and directly flash
// super without physically building and storing the image.
class SuperLayoutBuilder final {
public:
// Open a super_empty.img, return false on failure. This must be called to
// initialize the tool. If it returns false, either the image failed to
// parse, or the tool is not compatible with how the device is configured
// (in which case fastbootd should be preferred).
[[nodiscard]] bool Open(android::base::borrowed_fd fd);
[[nodiscard]] bool Open(const void* data, size_t bytes);
[[nodiscard]] bool Open(const LpMetadata& metadata);
// Add a partition's image and size to the work list. If false is returned,
// there was either a duplicate partition or not enough space in super.
bool AddPartition(const std::string& partition_name, const std::string& image_name,
uint64_t partition_size);
// Return the list of extents describing the super image. If this list is
// empty, then there was an unrecoverable error in building the list.
std::vector<SuperImageExtent> GetImageLayout();
// Return the current metadata.
std::unique_ptr<LpMetadata> Export() const { return builder_->Export(); }
private:
std::unique_ptr<MetadataBuilder> builder_;
std::unordered_map<std::string, std::string> image_map_;
};
std::ostream& operator<<(std::ostream& stream, const SuperImageExtent& extent);
} // namespace fs_mgr
} // namespace android

26
fs_mgr/liblp/liblp_test.xml
View File

@ -1,26 +0,0 @@
<?xml version="1.0" encoding="utf-8"?>
<!-- Copyright (C) 2019 The Android Open Source Project
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<configuration description="Config for liblp_test">
<target_preparer class="com.android.tradefed.targetprep.RootTargetPreparer"/>
<target_preparer class="com.android.tradefed.targetprep.PushFilePreparer">
<option name="cleanup" value="true" />
<option name="push" value="liblp_test->/data/local/tmp/liblp_test" />
</target_preparer>
<test class="com.android.tradefed.testtype.GTest" >
<option name="native-test-device-path" value="/data/local/tmp" />
<option name="module-name" value="liblp_test" />
</test>
</configuration>

241
fs_mgr/liblp/super_layout_builder.cpp Normal file
View File

@ -0,0 +1,241 @@
//
// Copyright (C) 2023 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
#include <liblp/super_layout_builder.h>
#include <liblp/liblp.h>
#include "images.h"
#include "utility.h"
#include "writer.h"
using android::base::borrowed_fd;
using android::base::unique_fd;
namespace android {
namespace fs_mgr {
bool SuperLayoutBuilder::Open(borrowed_fd fd) {
auto metadata = ReadFromImageFile(fd.get());
if (!metadata) {
return false;
}
return Open(*metadata.get());
}
bool SuperLayoutBuilder::Open(const void* data, size_t size) {
auto metadata = ReadFromImageBlob(data, size);
if (!metadata) {
return false;
}
return Open(*metadata.get());
}
bool SuperLayoutBuilder::Open(const LpMetadata& metadata) {
for (const auto& partition : metadata.partitions) {
if (partition.attributes & LP_PARTITION_ATTR_SLOT_SUFFIXED) {
// Retrofit devices are not supported.
return false;
}
if (!(partition.attributes & LP_PARTITION_ATTR_READONLY)) {
// Writable partitions are not supported.
return false;
}
}
if (!metadata.extents.empty()) {
// Partitions that already have extents are not supported (should
// never be true of super_empty.img).
return false;
}
if (metadata.block_devices.size() != 1) {
// Only one "super" is supported.
return false;
}
builder_ = MetadataBuilder::New(metadata);
return !!builder_;
}
bool SuperLayoutBuilder::AddPartition(const std::string& partition_name,
const std::string& image_name, uint64_t partition_size) {
auto p = builder_->FindPartition(partition_name);
if (!p) {
return false;
}
if (!builder_->ResizePartition(p, partition_size)) {
return false;
}
image_map_.emplace(partition_name, image_name);
return true;
}
// Fill the space between each extent, if any, with either a fill or dontcare
// extent. The caller constructs a sample extent to re-use.
static bool AddGapExtents(std::vector<SuperImageExtent>* extents, SuperImageExtent::Type gap_type) {
std::vector<SuperImageExtent> old = std::move(*extents);
std::sort(old.begin(), old.end());
*extents = {};
uint64_t current_offset = 0;
for (const auto& extent : old) {
// Check for overlapping extents - this would be a serious error.
if (current_offset > extent.offset) {
LOG(INFO) << "Overlapping extents detected; cannot layout temporary super image";
return false;
}
if (extent.offset != current_offset) {
uint64_t gap_size = extent.offset - current_offset;
extents->emplace_back(current_offset, gap_size, gap_type);
current_offset = extent.offset;
}
extents->emplace_back(extent);
current_offset += extent.size;
}
return true;
}
std::vector<SuperImageExtent> SuperLayoutBuilder::GetImageLayout() {
auto metadata = builder_->Export();
if (!metadata) {
return {};
}
std::vector<SuperImageExtent> extents;
// Write the primary and backup copies of geometry.
std::string geometry_bytes = SerializeGeometry(metadata->geometry);
auto blob = std::make_shared<std::string>(std::move(geometry_bytes));
extents.emplace_back(0, GetPrimaryGeometryOffset(), SuperImageExtent::Type::ZERO);
extents.emplace_back(GetPrimaryGeometryOffset(), blob);
extents.emplace_back(GetBackupGeometryOffset(), blob);
// Write the primary and backup copies of each metadata slot. When flashing,
// all metadata copies are the same, even for different slots.
std::string metadata_bytes = SerializeMetadata(*metadata.get());
// Align metadata size to 4KB. This makes the layout easily compatible with
// libsparse.
static constexpr size_t kSparseAlignment = 4096;
size_t metadata_aligned_bytes;
if (!AlignTo(metadata_bytes.size(), kSparseAlignment, &metadata_aligned_bytes)) {
LOG(ERROR) << "Unable to align metadata size " << metadata_bytes.size() << " to "
<< kSparseAlignment;
return {};
}
metadata_bytes.resize(metadata_aligned_bytes, '\0');
// However, alignment can cause larger-than-supported metadata blocks. Fall
// back to fastbootd/update-super.
if (metadata_bytes.size() > metadata->geometry.metadata_max_size) {
LOG(VERBOSE) << "Aligned metadata size " << metadata_bytes.size()
<< " is larger than maximum metadata size "
<< metadata->geometry.metadata_max_size;
return {};
}
blob = std::make_shared<std::string>(std::move(metadata_bytes));
for (uint32_t i = 0; i < metadata->geometry.metadata_slot_count; i++) {
int64_t metadata_primary = GetPrimaryMetadataOffset(metadata->geometry, i);
int64_t metadata_backup = GetBackupMetadataOffset(metadata->geometry, i);
extents.emplace_back(metadata_primary, blob);
extents.emplace_back(metadata_backup, blob);
}
// Add extents for each partition.
for (const auto& partition : metadata->partitions) {
auto partition_name = GetPartitionName(partition);
auto image_name_iter = image_map_.find(partition_name);
if (image_name_iter == image_map_.end()) {
if (partition.num_extents != 0) {
LOG(ERROR) << "Partition " << partition_name
<< " has extents but no image filename";
return {};
}
continue;
}
const auto& image_name = image_name_iter->second;
uint64_t image_offset = 0;
for (uint32_t i = 0; i < partition.num_extents; i++) {
const auto& e = metadata->extents[partition.first_extent_index + i];
if (e.target_type != LP_TARGET_TYPE_LINEAR) {
// Any type other than LINEAR isn't understood here. We don't even
// bother with ZERO, which is never generated.
LOG(INFO) << "Unknown extent type from liblp: " << e.target_type;
return {};
}
size_t size = e.num_sectors * LP_SECTOR_SIZE;
uint64_t super_offset = e.target_data * LP_SECTOR_SIZE;
extents.emplace_back(super_offset, size, image_name, image_offset);
image_offset += size;
}
}
if (!AddGapExtents(&extents, SuperImageExtent::Type::DONTCARE)) {
return {};
}
return extents;
}
bool SuperImageExtent::operator==(const SuperImageExtent& other) const {
if (offset != other.offset) {
return false;
}
if (size != other.size) {
return false;
}
if (type != other.type) {
return false;
}
switch (type) {
case Type::DATA:
return *blob == *other.blob;
case Type::PARTITION:
return image_name == other.image_name && image_offset == other.image_offset;
default:
return true;
}
}
std::ostream& operator<<(std::ostream& stream, const SuperImageExtent& extent) {
stream << "extent:" << extent.offset << ":" << extent.size << ":";
switch (extent.type) {
case SuperImageExtent::Type::DATA:
stream << "data";
break;
case SuperImageExtent::Type::PARTITION:
stream << "partition:" << extent.image_name << ":" << extent.image_offset;
break;
case SuperImageExtent::Type::ZERO:
stream << "zero";
break;
case SuperImageExtent::Type::DONTCARE:
stream << "dontcare";
break;
default:
stream << "invalid";
}
return stream;
}
} // namespace fs_mgr
} // namespace android

141
fs_mgr/liblp/super_layout_builder_test.cpp Normal file
View File

@ -0,0 +1,141 @@
//
// Copyright (C) 2023 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
#include <gmock/gmock.h>
#include <gtest/gtest.h>
#include <liblp/builder.h>
#include <liblp/super_layout_builder.h>
#include <storage_literals/storage_literals.h>
#include "images.h"
#include "writer.h"
using namespace android::fs_mgr;
using namespace android::storage_literals;
TEST(SuperImageTool, Layout) {
auto builder = MetadataBuilder::New(4_MiB, 8_KiB, 2);
ASSERT_NE(builder, nullptr);
Partition* p = builder->AddPartition("system_a", LP_PARTITION_ATTR_READONLY);
ASSERT_NE(p, nullptr);
auto metadata = builder->Export();
ASSERT_NE(metadata, nullptr);
SuperLayoutBuilder tool;
ASSERT_TRUE(tool.Open(*metadata.get()));
ASSERT_TRUE(tool.AddPartition("system_a", "system.img", 16_KiB));
// Get a copy of the metadata we'd expect if flashing.
ASSERT_TRUE(builder->ResizePartition(p, 16_KiB));
metadata = builder->Export();
ASSERT_NE(metadata, nullptr);
auto geometry_blob = std::make_shared<std::string>(SerializeGeometry(metadata->geometry));
auto metadata_blob = std::make_shared<std::string>(SerializeMetadata(*metadata.get()));
metadata_blob->resize(4_KiB, '\0');
auto extents = tool.GetImageLayout();
ASSERT_EQ(extents.size(), 12);
EXPECT_EQ(extents[0], SuperImageExtent(0, 4096, SuperImageExtent::Type::ZERO));
EXPECT_EQ(extents[1], SuperImageExtent(4096, geometry_blob));
EXPECT_EQ(extents[2], SuperImageExtent(8192, geometry_blob));
EXPECT_EQ(extents[3], SuperImageExtent(12288, metadata_blob));
EXPECT_EQ(extents[4], SuperImageExtent(16384, 4096, SuperImageExtent::Type::DONTCARE));
EXPECT_EQ(extents[5], SuperImageExtent(20480, metadata_blob));
EXPECT_EQ(extents[6], SuperImageExtent(24576, 4096, SuperImageExtent::Type::DONTCARE));
EXPECT_EQ(extents[7], SuperImageExtent(28672, metadata_blob));
EXPECT_EQ(extents[8], SuperImageExtent(32768, 4096, SuperImageExtent::Type::DONTCARE));
EXPECT_EQ(extents[9], SuperImageExtent(36864, metadata_blob));
EXPECT_EQ(extents[10], SuperImageExtent(40960, 4096, SuperImageExtent::Type::DONTCARE));
EXPECT_EQ(extents[11], SuperImageExtent(45056, 16384, "system.img", 0));
}
TEST(SuperImageTool, NoWritablePartitions) {
auto builder = MetadataBuilder::New(4_MiB, 8_KiB, 2);
ASSERT_NE(builder, nullptr);
Partition* p = builder->AddPartition("system_a", 0);
ASSERT_NE(p, nullptr);
auto metadata = builder->Export();
ASSERT_NE(metadata, nullptr);
SuperLayoutBuilder tool;
ASSERT_FALSE(tool.Open(*metadata.get()));
}
TEST(SuperImageTool, NoRetrofit) {
auto builder = MetadataBuilder::New(4_MiB, 8_KiB, 2);
ASSERT_NE(builder, nullptr);
Partition* p = builder->AddPartition("system_a", LP_PARTITION_ATTR_READONLY);
ASSERT_NE(p, nullptr);
auto metadata = builder->Export();
ASSERT_NE(metadata, nullptr);
// Add an extra block device.
metadata->block_devices.emplace_back(metadata->block_devices[0]);
SuperLayoutBuilder tool;
ASSERT_FALSE(tool.Open(*metadata.get()));
}
TEST(SuperImageTool, NoRetrofit2) {
auto builder = MetadataBuilder::New(4_MiB, 8_KiB, 2);
ASSERT_NE(builder, nullptr);
Partition* p = builder->AddPartition(
"system_a", LP_PARTITION_ATTR_READONLY | LP_PARTITION_ATTR_SLOT_SUFFIXED);
ASSERT_NE(p, nullptr);
auto metadata = builder->Export();
ASSERT_NE(metadata, nullptr);
SuperLayoutBuilder tool;
ASSERT_FALSE(tool.Open(*metadata.get()));
}
TEST(SuperImageTool, NoFixedPartitions) {
auto builder = MetadataBuilder::New(4_MiB, 8_KiB, 2);
ASSERT_NE(builder, nullptr);
Partition* p = builder->AddPartition("system_a", LP_PARTITION_ATTR_READONLY);
ASSERT_NE(p, nullptr);
ASSERT_TRUE(builder->ResizePartition(p, 4_KiB));
auto metadata = builder->Export();
ASSERT_NE(metadata, nullptr);
SuperLayoutBuilder tool;
ASSERT_FALSE(tool.Open(*metadata.get()));
}
TEST(SuperImageTool, LargeAlignedMetadata) {
auto builder = MetadataBuilder::New(4_MiB, 512, 2);
ASSERT_NE(builder, nullptr);
auto metadata = builder->Export();
ASSERT_NE(metadata, nullptr);
SuperLayoutBuilder tool;
ASSERT_TRUE(tool.Open(*metadata.get()));
auto extents = tool.GetImageLayout();
ASSERT_TRUE(extents.empty());
}