Merge "linker: align allocated blocks to 16 bytes"

linker/linker_allocator.cpp

@@ -189,6 +189,8 @@ void LinkerSmallObjectAllocator::create_page_record(void* page_addr, size_t free
 }
 
 void LinkerSmallObjectAllocator::alloc_page() {
+  static_assert(sizeof(page_info) % 16 == 0,
+                "sizeof(page_info) is not multiple of 16");
   void* map_ptr = mmap(nullptr, PAGE_SIZE,
       PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, 0, 0);
   if (map_ptr == MAP_FAILED) {

linker/linker_allocator.h

@@ -45,7 +45,7 @@ struct page_info {
     // and allocator_addr for small ones.
     LinkerSmallObjectAllocator* allocator_addr;
   };
-};
+} __attribute__((aligned(16)));
 
 struct small_object_page_record {
   void* page_addr;

linker/linker_block_allocator.cpp

@@ -22,9 +22,14 @@
 
 #include "private/bionic_prctl.h"
 
+// the multiplier should be power of 2
+static constexpr size_t round_up(size_t size, size_t multiplier) {
+  return (size + (multiplier - 1)) & ~(multiplier-1);
+}
+
 struct LinkerBlockAllocatorPage {
   LinkerBlockAllocatorPage* next;
-  uint8_t bytes[PAGE_SIZE-sizeof(LinkerBlockAllocatorPage*)];
+  uint8_t bytes[PAGE_SIZE - 16] __attribute__((aligned(16)));
 };
 
 struct FreeBlockInfo {
@@ -33,7 +38,8 @@ struct FreeBlockInfo {
 };
 
 LinkerBlockAllocator::LinkerBlockAllocator(size_t block_size)
-  : block_size_(block_size < sizeof(FreeBlockInfo) ? sizeof(FreeBlockInfo) : block_size),
+  : block_size_(
+      round_up(block_size < sizeof(FreeBlockInfo) ? sizeof(FreeBlockInfo) : block_size, 16)),
     page_list_(nullptr),
     free_block_list_(nullptr)
 {}
@@ -95,6 +101,9 @@ void LinkerBlockAllocator::protect_all(int prot) {
 }
 
 void LinkerBlockAllocator::create_new_page() {
+  static_assert(sizeof(LinkerBlockAllocatorPage) == PAGE_SIZE,
+                "Invalid sizeof(LinkerBlockAllocatorPage)");
+
   LinkerBlockAllocatorPage* page = reinterpret_cast<LinkerBlockAllocatorPage*>(
       mmap(nullptr, PAGE_SIZE, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, 0, 0));
 

linker/tests/linker_block_allocator_test.cpp

@@ -53,10 +53,12 @@ TEST(linker_allocator, test_nominal) {
 
   test_struct_nominal* ptr1 = allocator.alloc();
   ASSERT_TRUE(ptr1 != nullptr);
+  ASSERT_EQ(0U, reinterpret_cast<uintptr_t>(ptr1) % 16);
   test_struct_nominal* ptr2 = allocator.alloc();
+  ASSERT_EQ(0U, reinterpret_cast<uintptr_t>(ptr2) % 16);
   ASSERT_TRUE(ptr2 != nullptr);
   // they should be next to each other.
-  ASSERT_EQ(ptr1+1, ptr2);
+  ASSERT_EQ(reinterpret_cast<uint8_t*>(ptr1)+16, reinterpret_cast<uint8_t*>(ptr2));
 
   ptr1->value = 42;
 
@@ -71,8 +73,10 @@ TEST(linker_allocator, test_small) {
   char* ptr2 = reinterpret_cast<char*>(allocator.alloc());
 
   ASSERT_TRUE(ptr1 != nullptr);
+  ASSERT_EQ(0U, reinterpret_cast<uintptr_t>(ptr1) % 16);
   ASSERT_TRUE(ptr2 != nullptr);
-  ASSERT_EQ(ptr1+2*sizeof(void*), ptr2);
+  ASSERT_EQ(0U, reinterpret_cast<uintptr_t>(ptr2) % 16);
+  ASSERT_EQ(ptr1+16, ptr2); // aligned to 16
 }
 
 TEST(linker_allocator, test_larger) {
@@ -82,9 +86,11 @@ TEST(linker_allocator, test_larger) {
   test_struct_larger* ptr2 = allocator.alloc();
 
   ASSERT_TRUE(ptr1 != nullptr);
+  ASSERT_EQ(0U, reinterpret_cast<uintptr_t>(ptr1) % 16);
   ASSERT_TRUE(ptr2 != nullptr);
+  ASSERT_EQ(0U, reinterpret_cast<uintptr_t>(ptr2) % 16);
 
-  ASSERT_EQ(ptr1+1, ptr2);
+  ASSERT_EQ(reinterpret_cast<uint8_t*>(ptr1) + 1024, reinterpret_cast<uint8_t*>(ptr2));
 
   // lets allocate until we reach next page.
   size_t n = kPageSize/sizeof(test_struct_larger) + 1 - 2;

linker/tests/linker_memory_allocator_test.cpp

@@ -96,6 +96,7 @@ TEST(linker_memory, test_realloc) {
 
   ASSERT_TRUE(reallocated_ptr != nullptr);
   ASSERT_TRUE(reallocated_ptr != array);
+  ASSERT_EQ(0U, reinterpret_cast<uintptr_t>(reallocated_ptr) % 16);
 
   ASSERT_TRUE(memcmp(reallocated_ptr, model, array_size * 2) == 0);
 
@@ -107,6 +108,7 @@ TEST(linker_memory, test_realloc) {
 
   ASSERT_TRUE(reallocated_ptr != nullptr);
   ASSERT_TRUE(reallocated_ptr != array);
+  ASSERT_EQ(0U, reinterpret_cast<uintptr_t>(reallocated_ptr) % 16);
 
   ASSERT_TRUE(memcmp(reallocated_ptr, model, 4000) == 0);
 
@@ -125,7 +127,10 @@ TEST(linker_memory, test_small_smoke) {
       reinterpret_cast<test_struct_small*>(allocator.alloc(sizeof(test_struct_small)));
 
   ASSERT_TRUE(ptr1 != nullptr);
+  ASSERT_EQ(0U, reinterpret_cast<uintptr_t>(ptr1) % 16);
   ASSERT_TRUE(ptr2 != nullptr);
+  ASSERT_EQ(0U, reinterpret_cast<uintptr_t>(ptr2) % 16);
+
   ASSERT_EQ(reinterpret_cast<uintptr_t>(ptr1)+16, reinterpret_cast<uintptr_t>(ptr2));
   ASSERT_TRUE(memcmp(ptr1, zeros, 16) == 0);
 
@@ -143,7 +148,9 @@ TEST(linker_memory, test_huge_smoke) {
       reinterpret_cast<test_struct_huge*>(allocator.alloc(sizeof(test_struct_huge)));
 
   ASSERT_TRUE(ptr1 != nullptr);
+  ASSERT_EQ(0U, reinterpret_cast<uintptr_t>(ptr1) % 16);
   ASSERT_TRUE(ptr2 != nullptr);
+  ASSERT_EQ(0U, reinterpret_cast<uintptr_t>(ptr2) % 16);
 
   ASSERT_TRUE(
       reinterpret_cast<uintptr_t>(ptr1)/kPageSize != reinterpret_cast<uintptr_t>(ptr2)/kPageSize);
@@ -160,7 +167,9 @@ TEST(linker_memory, test_large) {
       reinterpret_cast<test_struct_large*>(allocator.alloc(1024));
 
   ASSERT_TRUE(ptr1 != nullptr);
+  ASSERT_EQ(0U, reinterpret_cast<uintptr_t>(ptr1) % 16);
   ASSERT_TRUE(ptr2 != nullptr);
+  ASSERT_EQ(0U, reinterpret_cast<uintptr_t>(ptr2) % 16);
 
   ASSERT_EQ(reinterpret_cast<uintptr_t>(ptr1) + 1024, reinterpret_cast<uintptr_t>(ptr2));
 
@@ -179,6 +188,7 @@ TEST(linker_memory, test_large) {
       reinterpret_cast<test_struct_large*>(allocator.alloc(sizeof(test_struct_large)));
 
   ASSERT_TRUE(ptr_to_free != nullptr);
+  ASSERT_EQ(0U, reinterpret_cast<uintptr_t>(ptr_to_free) % 16);
 
   allocator.free(ptr1);