pa-fazil
/
android_bionic
mirror of https://github.com/fazilsheik96/android_bionic.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Reduce LinkerSmallObjectAllocator memory overhead 

The current implementation of LinkerSmallObjectAllocator keeps record
of pages in a vector, which uses its own page(s).  This is at least a
page overhead per LinkerSmallObjectAllocator.

This change removes the page record vector by managing the pages in a
doubly linked list.

We also fix a bug where we are actually keeping up to 2 free pages
instead of just one.

The memory used by small objects when running 'dd', before this change:
    72 KB  [anon:linker_alloc_small_objects]
    28 KB  [anon:linker_alloc_vector]

After this change:
    60 KB  [anon:linker_alloc_small_objects]

Test: Boot cuttlefish and check memory used by linker.
Change-Id: I3468fa4d853c78b4bc02bfb84a3531653f74fb17
This commit is contained in:
Vic Yang 2018-12-02 23:46:26 -08:00
parent c49776bffc
commit 5493851e1b
2 changed files with 131 additions and 143 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

206
linker/linker_allocator.cpp
View File

@ -30,9 +30,6 @@
#include "linker_debug.h"
#include "linker.h"
#include <algorithm>
#include <vector>
#include <stdlib.h>
#include <sys/mman.h>
#include <sys/prctl.h>
@ -56,9 +53,19 @@
// the memory.
//
// For a pointer allocated using SmallObjectAllocator it adds
// the block to free_blocks_list_. If the number of free pages reaches 2,
// SmallObjectAllocator munmaps one of the pages keeping the other one
// in reserve.
// the block to free_blocks_list in the corresponding page. If the number of
// free pages reaches 2, SmallObjectAllocator munmaps one of the pages keeping
// the other one in reserve.
// Memory management for large objects is fairly straightforward, but for small
// objects it is more complicated. If you are changing this code, one simple
// way to evaluate the memory usage change is by running 'dd' and examine the
// memory usage by 'showmap $(pidof dd)'. 'dd' is nice in that:
// 1. It links in quite a few libraries, so you get some linker memory use.
// 2. When run with no arguments, it sits waiting for input, so it is easy to
// examine its memory usage with showmap.
// 3. Since it does nothing while waiting for input, the memory usage is
// determinisitic.
static const char kSignature[4] = {'L', 'M', 'A', 1};
@ -67,10 +74,6 @@ static const size_t kSmallObjectMaxSize = 1 << kSmallObjectMaxSizeLog2;
// This type is used for large allocations (with size >1k)
static const uint32_t kLargeObject = 111;
bool operator<(const small_object_page_record& one, const small_object_page_record& two) {
return one.page_addr < two.page_addr;
}
static inline uint16_t log2(size_t number) {
uint16_t result = 0;
number--;
@ -83,149 +86,164 @@ static inline uint16_t log2(size_t number) {
return result;
}
LinkerSmallObjectAllocator::LinkerSmallObjectAllocator(uint32_t type, size_t block_size)
: type_(type), block_size_(block_size), free_pages_cnt_(0), free_blocks_list_(nullptr) {}
LinkerSmallObjectAllocator::LinkerSmallObjectAllocator(uint32_t type,
size_t block_size)
: type_(type),
block_size_(block_size),
free_pages_cnt_(0),
page_list_(nullptr) {}
void* LinkerSmallObjectAllocator::alloc() {
CHECK(block_size_ != 0);
if (free_blocks_list_ == nullptr) {
if (page_list_ == nullptr) {
alloc_page();
}
small_object_block_record* block_record = free_blocks_list_;
// Fully allocated pages are de-managed and removed from the page list, so
// every page from the page list must be useable. Let's just take the first
// one.
small_object_page_info* page = page_list_;
CHECK(page->free_block_list != nullptr);
small_object_block_record* const block_record = page->free_block_list;
if (block_record->free_blocks_cnt > 1) {
small_object_block_record* next_free = reinterpret_cast<small_object_block_record*>(
reinterpret_cast<uint8_t*>(block_record) + block_size_);
small_object_block_record* next_free =
reinterpret_cast<small_object_block_record*>(
reinterpret_cast<uint8_t*>(block_record) + block_size_);
next_free->next = block_record->next;
next_free->free_blocks_cnt = block_record->free_blocks_cnt - 1;
free_blocks_list_ = next_free;
page->free_block_list = next_free;
} else {
free_blocks_list_ = block_record->next;
page->free_block_list = block_record->next;
}
// bookkeeping...
auto page_record = find_page_record(block_record);
if (page_record->allocated_blocks_cnt == 0) {
if (page->allocated_blocks_cnt == 0) {
free_pages_cnt_--;
}
page_record->free_blocks_cnt--;
page_record->allocated_blocks_cnt++;
page->free_blocks_cnt--;
page->allocated_blocks_cnt++;
memset(block_record, 0, block_size_);
if (page->free_blocks_cnt == 0) {
// De-manage fully allocated pages. These pages will be managed again if
// a block is freed.
remove_from_page_list(page);
}
return block_record;
}
void LinkerSmallObjectAllocator::free_page(linker_vector_t::iterator page_record) {
void* page_start = reinterpret_cast<void*>(page_record->page_addr);
void* page_end = reinterpret_cast<void*>(reinterpret_cast<uintptr_t>(page_start) + PAGE_SIZE);
while (free_blocks_list_ != nullptr &&
free_blocks_list_ > page_start &&
free_blocks_list_ < page_end) {
free_blocks_list_ = free_blocks_list_->next;
void LinkerSmallObjectAllocator::free_page(small_object_page_info* page) {
CHECK(page->allocated_blocks_cnt == 0);
if (page->prev_page) {
page->prev_page->next_page = page->next_page;
}
small_object_block_record* current = free_blocks_list_;
while (current != nullptr) {
while (current->next > page_start && current->next < page_end) {
current->next = current->next->next;
}
current = current->next;
if (page->next_page) {
page->next_page->prev_page = page->prev_page;
}
munmap(page_start, PAGE_SIZE);
page_records_.erase(page_record);
if (page_list_ == page) {
page_list_ = page->next_page;
}
munmap(page, PAGE_SIZE);
free_pages_cnt_--;
}
void LinkerSmallObjectAllocator::free(void* ptr) {
auto page_record = find_page_record(ptr);
small_object_page_info* const page =
reinterpret_cast<small_object_page_info*>(
PAGE_START(reinterpret_cast<uintptr_t>(ptr)));
ssize_t offset = reinterpret_cast<uintptr_t>(ptr) - sizeof(page_info);
const ssize_t offset =
reinterpret_cast<uintptr_t>(ptr) - sizeof(small_object_page_info);
if (offset % block_size_ != 0) {
async_safe_fatal("invalid pointer: %p (block_size=%zd)", ptr, block_size_);
}
memset(ptr, 0, block_size_);
small_object_block_record* block_record = reinterpret_cast<small_object_block_record*>(ptr);
small_object_block_record* const block_record =
reinterpret_cast<small_object_block_record*>(ptr);
block_record->next = free_blocks_list_;
block_record->next = page->free_block_list;
block_record->free_blocks_cnt = 1;
free_blocks_list_ = block_record;
page->free_block_list = block_record;
page->free_blocks_cnt++;
page->allocated_blocks_cnt--;
page_record->free_blocks_cnt++;
page_record->allocated_blocks_cnt--;
if (page_record->allocated_blocks_cnt == 0) {
if (free_pages_cnt_++ > 1) {
if (page->allocated_blocks_cnt == 0) {
if (++free_pages_cnt_ > 1) {
// if we already have a free page - unmap this one.
free_page(page_record);
free_page(page);
}
} else if (page->free_blocks_cnt == 1) {
// We just freed from a full page. Add this page back to the list.
add_to_page_list(page);
}
}
linker_vector_t::iterator LinkerSmallObjectAllocator::find_page_record(void* ptr) {
void* addr = reinterpret_cast<void*>(PAGE_START(reinterpret_cast<uintptr_t>(ptr)));
small_object_page_record boundary;
boundary.page_addr = addr;
linker_vector_t::iterator it = std::lower_bound(
page_records_.begin(), page_records_.end(), boundary);
if (it == page_records_.end() || it->page_addr != addr) {
// not found...
async_safe_fatal("page record for %p was not found (block_size=%zd)", ptr, block_size_);
}
return it;
}
void LinkerSmallObjectAllocator::create_page_record(void* page_addr, size_t free_blocks_cnt) {
small_object_page_record record;
record.page_addr = page_addr;
record.free_blocks_cnt = free_blocks_cnt;
record.allocated_blocks_cnt = 0;
linker_vector_t::iterator it = std::lower_bound(
page_records_.begin(), page_records_.end(), record);
page_records_.insert(it, record);
}
void LinkerSmallObjectAllocator::alloc_page() {
static_assert(sizeof(page_info) % 16 == 0, "sizeof(page_info) is not multiple of 16");
void* map_ptr = mmap(nullptr, PAGE_SIZE, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0);
static_assert(sizeof(small_object_page_info) % 16 == 0,
"sizeof(small_object_page_info) is not multiple of 16");
void* const map_ptr = mmap(nullptr, PAGE_SIZE, PROT_READ | PROT_WRITE,
MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
if (map_ptr == MAP_FAILED) {
async_safe_fatal("mmap failed: %s", strerror(errno));
}
prctl(PR_SET_VMA, PR_SET_VMA_ANON_NAME, map_ptr, PAGE_SIZE, "linker_alloc_small_objects");
prctl(PR_SET_VMA, PR_SET_VMA_ANON_NAME, map_ptr, PAGE_SIZE,
"linker_alloc_small_objects");
page_info* info = reinterpret_cast<page_info*>(map_ptr);
memcpy(info->signature, kSignature, sizeof(kSignature));
info->type = type_;
info->allocator_addr = this;
small_object_page_info* const page =
reinterpret_cast<small_object_page_info*>(map_ptr);
memcpy(page->info.signature, kSignature, sizeof(kSignature));
page->info.type = type_;
page->info.allocator_addr = this;
size_t free_blocks_cnt = (PAGE_SIZE - sizeof(page_info))/block_size_;
const size_t free_blocks_cnt =
(PAGE_SIZE - sizeof(small_object_page_info)) / block_size_;
create_page_record(map_ptr, free_blocks_cnt);
page->free_blocks_cnt = free_blocks_cnt;
page->allocated_blocks_cnt = 0;
small_object_block_record* first_block = reinterpret_cast<small_object_block_record*>(info + 1);
first_block->next = free_blocks_list_;
small_object_block_record* const first_block =
reinterpret_cast<small_object_block_record*>(page + 1);
first_block->next = nullptr;
first_block->free_blocks_cnt = free_blocks_cnt;
free_blocks_list_ = first_block;
page->free_block_list = first_block;
add_to_page_list(page);
free_pages_cnt_++;
}
void LinkerSmallObjectAllocator::add_to_page_list(small_object_page_info* page) {
page->next_page = page_list_;
page->prev_page = nullptr;
if (page_list_) {
page_list_->prev_page = page;
}
page_list_ = page;
}
void LinkerSmallObjectAllocator::remove_from_page_list(
small_object_page_info* page) {
if (page->prev_page) {
page->prev_page->next_page = page->next_page;
}
if (page->next_page) {
page->next_page->prev_page = page->prev_page;
}
if (page_list_ == page) {
page_list_ = page->next_page;
}
page->prev_page = nullptr;
page->next_page = nullptr;
}
void LinkerMemoryAllocator::initialize_allocators() {
if (allocators_ != nullptr) {

68
linker/linker_allocator.h
View File

@ -36,8 +36,6 @@
#include <stddef.h>
#include <unistd.h>
#include <vector>
#include <async_safe/log.h>
const uint32_t kSmallObjectMaxSizeLog2 = 10;
@ -59,57 +57,31 @@ struct page_info {
};
} __attribute__((aligned(16)));
struct small_object_page_record {
void* page_addr;
size_t free_blocks_cnt;
size_t allocated_blocks_cnt;
};
// for lower_bound...
bool operator<(const small_object_page_record& one, const small_object_page_record& two);
struct small_object_block_record {
small_object_block_record* next;
size_t free_blocks_cnt;
};
// This is implementation for std::vector allocator
template <typename T>
class linker_vector_allocator {
public:
typedef T value_type;
typedef T* pointer;
typedef const T* const_pointer;
typedef T& reference;
typedef const T& const_reference;
typedef size_t size_type;
typedef ptrdiff_t difference_type;
// This structure is placed at the beginning of each page managed by
// LinkerSmallObjectAllocator. Note that a page_info struct is expected at the
// beginning of each page as well, and therefore this structure contains a
// page_info as its *first* field.
struct small_object_page_info {
page_info info; // Must be the first field.
T* allocate(size_t n, const T* hint = nullptr) {
size_t size = n * sizeof(T);
void* ptr = mmap(const_cast<T*>(hint), size, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS,
-1, 0);
if (ptr == MAP_FAILED) {
// Spec says we need to throw std::bad_alloc here but because our
// code does not support exception handling anyways - we are going to abort.
async_safe_fatal("mmap failed: %s", strerror(errno));
}
// Doubly linked list for traversing all pages allocated by a
// LinkerSmallObjectAllocator.
small_object_page_info* next_page;
small_object_page_info* prev_page;
prctl(PR_SET_VMA, PR_SET_VMA_ANON_NAME, ptr, size, "linker_alloc_vector");
// Linked list containing all free blocks in this page.
small_object_block_record* free_block_list;
return reinterpret_cast<T*>(ptr);
}
void deallocate(T* ptr, size_t n) {
munmap(ptr, n * sizeof(T));
}
// Free/allocated blocks counter.
size_t free_blocks_cnt;
size_t allocated_blocks_cnt;
};
typedef
std::vector<small_object_page_record, linker_vector_allocator<small_object_page_record>>
linker_vector_t;
class LinkerSmallObjectAllocator {
public:
LinkerSmallObjectAllocator(uint32_t type, size_t block_size);
@ -119,18 +91,16 @@ class LinkerSmallObjectAllocator {
size_t get_block_size() const { return block_size_; }
private:
void alloc_page();
void free_page(linker_vector_t::iterator page_record);
linker_vector_t::iterator find_page_record(void* ptr);
void create_page_record(void* page_addr, size_t free_blocks_cnt);
void free_page(small_object_page_info* page);
void add_to_page_list(small_object_page_info* page);
void remove_from_page_list(small_object_page_info* page);
uint32_t type_;
size_t block_size_;
size_t free_pages_cnt_;
small_object_block_record* free_blocks_list_;
// sorted vector of page records
linker_vector_t page_records_;
small_object_page_info* page_list_;
};
class LinkerMemoryAllocator {