From 3c8fc2fea9dac044f4903b0c315b5cda1c1f5301 Mon Sep 17 00:00:00 2001 From: Josh Gao Date: Thu, 8 Oct 2015 14:49:26 -0700 Subject: [PATCH] Move malloc dispatch table to __libc_globals. Change-Id: Ic20b980d1e8b6c2d4b773ebe336658fd17c737cb --- libc/bionic/libc_init_dynamic.cpp | 4 +- libc/bionic/malloc_debug_common.cpp | 123 +++++++++++++++++--------- libc/bionic/malloc_debug_common.h | 34 +------ libc/private/bionic_globals.h | 4 +- libc/private/bionic_malloc_dispatch.h | 68 ++++++++++++++ 5 files changed, 153 insertions(+), 80 deletions(-) create mode 100644 libc/private/bionic_malloc_dispatch.h diff --git a/libc/bionic/libc_init_dynamic.cpp b/libc/bionic/libc_init_dynamic.cpp index 3bb6e8939..edf6a4444 100644 --- a/libc/bionic/libc_init_dynamic.cpp +++ b/libc/bionic/libc_init_dynamic.cpp @@ -50,11 +50,11 @@ #include #include "libc_init_common.h" +#include "private/bionic_globals.h" #include "private/bionic_tls.h" #include "private/KernelArgumentBlock.h" extern "C" { - extern void malloc_debug_init(void); extern void malloc_debug_fini(void); extern void netdClientInit(void); extern int __cxa_atexit(void (*)(void *), void *, void *); @@ -78,7 +78,7 @@ __attribute__((constructor)) static void __libc_preinit() { __libc_init_common(*args); // Hooks for various libraries to let them know that we're starting up. - malloc_debug_init(); + __libc_globals.mutate(__libc_init_malloc); netdClientInit(); } diff --git a/libc/bionic/malloc_debug_common.cpp b/libc/bionic/malloc_debug_common.cpp index 5b7c42caa..9f7f3ba53 100644 --- a/libc/bionic/malloc_debug_common.cpp +++ b/libc/bionic/malloc_debug_common.cpp @@ -44,6 +44,7 @@ #include #include +#include "private/bionic_globals.h" #include "private/ScopedPthreadMutexLocker.h" #if defined(USE_JEMALLOC) @@ -56,33 +57,29 @@ #error "Either one of USE_DLMALLOC or USE_JEMALLOC must be defined." #endif +static constexpr MallocDebug __libc_malloc_default_dispatch + __attribute__((unused)) = { + Malloc(calloc), + Malloc(free), + Malloc(mallinfo), + Malloc(malloc), + Malloc(malloc_usable_size), + Malloc(memalign), + Malloc(posix_memalign), +#if defined(HAVE_DEPRECATED_MALLOC_FUNCS) + Malloc(pvalloc), +#endif + Malloc(realloc), +#if defined(HAVE_DEPRECATED_MALLOC_FUNCS) + Malloc(valloc), +#endif + }; + // In a VM process, this is set to 1 after fork()ing out of zygote. int gMallocLeakZygoteChild = 0; static HashTable g_hash_table; -// Support for malloc debugging. -// Table for dispatching malloc calls, initialized with default dispatchers. -static const MallocDebug __libc_malloc_default_dispatch __attribute__((aligned(32))) = { - Malloc(calloc), - Malloc(free), - Malloc(mallinfo), - Malloc(malloc), - Malloc(malloc_usable_size), - Malloc(memalign), - Malloc(posix_memalign), -#if defined(HAVE_DEPRECATED_MALLOC_FUNCS) - Malloc(pvalloc), -#endif - Malloc(realloc), -#if defined(HAVE_DEPRECATED_MALLOC_FUNCS) - Malloc(valloc), -#endif -}; - -// Selector of dispatch table to use for dispatching malloc calls. -static const MallocDebug* __libc_malloc_dispatch = &__libc_malloc_default_dispatch; - // Handle to shared library where actual memory allocation is implemented. // This library is loaded and memory allocation calls are redirected there // when libc.debug.malloc environment variable contains value other than @@ -244,46 +241,87 @@ extern "C" void free_malloc_leak_info(uint8_t* info) { // Allocation functions // ============================================================================= extern "C" void* calloc(size_t n_elements, size_t elem_size) { - return __libc_malloc_dispatch->calloc(n_elements, elem_size); + auto _calloc = __libc_globals->malloc_dispatch.calloc; + if (__predict_false(_calloc != nullptr)) { + return _calloc(n_elements, elem_size); + } + return Malloc(calloc)(n_elements, elem_size); } extern "C" void free(void* mem) { - __libc_malloc_dispatch->free(mem); + auto _free = __libc_globals->malloc_dispatch.free; + if (__predict_false(_free != nullptr)) { + _free(mem); + } else { + Malloc(free)(mem); + } } extern "C" struct mallinfo mallinfo() { - return __libc_malloc_dispatch->mallinfo(); + auto _mallinfo = __libc_globals->malloc_dispatch.mallinfo; + if (__predict_false(_mallinfo != nullptr)) { + return _mallinfo(); + } + return Malloc(mallinfo)(); } extern "C" void* malloc(size_t bytes) { - return __libc_malloc_dispatch->malloc(bytes); + auto _malloc = __libc_globals->malloc_dispatch.malloc; + if (__predict_false(_malloc != nullptr)) { + return _malloc(bytes); + } + return Malloc(malloc)(bytes); } extern "C" size_t malloc_usable_size(const void* mem) { - return __libc_malloc_dispatch->malloc_usable_size(mem); + auto _malloc_usable_size = __libc_globals->malloc_dispatch.malloc_usable_size; + if (__predict_false(_malloc_usable_size != nullptr)) { + return _malloc_usable_size(mem); + } + return Malloc(malloc_usable_size)(mem); } extern "C" void* memalign(size_t alignment, size_t bytes) { - return __libc_malloc_dispatch->memalign(alignment, bytes); + auto _memalign = __libc_globals->malloc_dispatch.memalign; + if (__predict_false(_memalign != nullptr)) { + return _memalign(alignment, bytes); + } + return Malloc(memalign)(alignment, bytes); } extern "C" int posix_memalign(void** memptr, size_t alignment, size_t size) { - return __libc_malloc_dispatch->posix_memalign(memptr, alignment, size); + auto _posix_memalign = __libc_globals->malloc_dispatch.posix_memalign; + if (__predict_false(_posix_memalign != nullptr)) { + return _posix_memalign(memptr, alignment, size); + } + return Malloc(posix_memalign)(memptr, alignment, size); } #if defined(HAVE_DEPRECATED_MALLOC_FUNCS) extern "C" void* pvalloc(size_t bytes) { - return __libc_malloc_dispatch->pvalloc(bytes); + auto _pvalloc = __libc_globals->malloc_dispatch.pvalloc; + if (__predict_false(_pvalloc != nullptr)) { + return _pvalloc(bytes); + } + return Malloc(pvalloc)(bytes); } #endif -extern "C" void* realloc(void* oldMem, size_t bytes) { - return __libc_malloc_dispatch->realloc(oldMem, bytes); +extern "C" void* realloc(void* old_mem, size_t bytes) { + auto _realloc = __libc_globals->malloc_dispatch.realloc; + if (__predict_false(_realloc != nullptr)) { + return _realloc(old_mem, bytes); + } + return Malloc(realloc)(old_mem, bytes); } #if defined(HAVE_DEPRECATED_MALLOC_FUNCS) extern "C" void* valloc(size_t bytes) { - return __libc_malloc_dispatch->valloc(bytes); + auto _valloc = __libc_globals->malloc_dispatch.valloc; + if (__predict_false(_valloc != nullptr)) { + return _valloc(bytes); + } + return Malloc(valloc)(bytes); } #endif @@ -326,7 +364,7 @@ static void InitMalloc(void* malloc_impl_handler, MallocDebug* table, const char } // Initializes memory allocation framework once per process. -static void malloc_init_impl() { +static void malloc_init_impl(libc_globals* globals) { const char* so_name = NULL; MallocDebugInit malloc_debug_initialize = NULL; unsigned int qemu_running = 0; @@ -442,7 +480,7 @@ static void malloc_init_impl() { // No need to init the dispatch table because we can only get // here if debug level is 1, 5, 10, or 20. - static MallocDebug malloc_dispatch_table __attribute__((aligned(32))); + MallocDebug malloc_dispatch_table; switch (g_malloc_debug_level) { case 1: InitMalloc(malloc_impl_handle, &malloc_dispatch_table, "leak"); @@ -480,7 +518,7 @@ static void malloc_init_impl() { getprogname(), g_malloc_debug_level); dlclose(malloc_impl_handle); } else { - __libc_malloc_dispatch = &malloc_dispatch_table; + globals->malloc_dispatch = malloc_dispatch_table; libc_malloc_impl_handle = malloc_impl_handle; } } @@ -507,15 +545,12 @@ static void malloc_fini_impl() { #endif // !LIBC_STATIC // Initializes memory allocation framework. -// This routine is called from __libc_init routines implemented -// in libc_init_static.c and libc_init_dynamic.c files. -extern "C" __LIBC_HIDDEN__ void malloc_debug_init() { +// This routine is called from __libc_init routines in libc_init_dynamic.cpp. +__LIBC_HIDDEN__ void __libc_init_malloc(libc_globals* globals) { + (void)globals; #if !defined(LIBC_STATIC) - static pthread_once_t malloc_init_once_ctl = PTHREAD_ONCE_INIT; - if (pthread_once(&malloc_init_once_ctl, malloc_init_impl)) { - error_log("Unable to initialize malloc_debug component."); - } -#endif // !LIBC_STATIC + malloc_init_impl(globals); +#endif } extern "C" __LIBC_HIDDEN__ void malloc_debug_fini() { diff --git a/libc/bionic/malloc_debug_common.h b/libc/bionic/malloc_debug_common.h index 5c73da31a..f8745da7f 100644 --- a/libc/bionic/malloc_debug_common.h +++ b/libc/bionic/malloc_debug_common.h @@ -38,6 +38,7 @@ #include #include "private/bionic_config.h" +#include "private/bionic_malloc_dispatch.h" #include "private/libc_logging.h" #define HASHTABLE_SIZE 1543 @@ -72,39 +73,6 @@ struct HashTable { HashEntry* slots[HASHTABLE_SIZE]; }; -/* Entry in malloc dispatch table. */ -typedef void* (*MallocDebugCalloc)(size_t, size_t); -typedef void (*MallocDebugFree)(void*); -typedef struct mallinfo (*MallocDebugMallinfo)(); -typedef void* (*MallocDebugMalloc)(size_t); -typedef size_t (*MallocDebugMallocUsableSize)(const void*); -typedef void* (*MallocDebugMemalign)(size_t, size_t); -typedef int (*MallocDebugPosixMemalign)(void**, size_t, size_t); -#if defined(HAVE_DEPRECATED_MALLOC_FUNCS) -typedef void* (*MallocDebugPvalloc)(size_t); -#endif -typedef void* (*MallocDebugRealloc)(void*, size_t); -#if defined(HAVE_DEPRECATED_MALLOC_FUNCS) -typedef void* (*MallocDebugValloc)(size_t); -#endif - -struct MallocDebug { - MallocDebugCalloc calloc; - MallocDebugFree free; - MallocDebugMallinfo mallinfo; - MallocDebugMalloc malloc; - MallocDebugMallocUsableSize malloc_usable_size; - MallocDebugMemalign memalign; - MallocDebugPosixMemalign posix_memalign; -#if defined(HAVE_DEPRECATED_MALLOC_FUNCS) - MallocDebugPvalloc pvalloc; -#endif - MallocDebugRealloc realloc; -#if defined(HAVE_DEPRECATED_MALLOC_FUNCS) - MallocDebugValloc valloc; -#endif -}; - typedef bool (*MallocDebugInit)(HashTable*, const MallocDebug*); typedef void (*MallocDebugFini)(int); diff --git a/libc/private/bionic_globals.h b/libc/private/bionic_globals.h index 644b5a4d0..a671d77bd 100644 --- a/libc/private/bionic_globals.h +++ b/libc/private/bionic_globals.h @@ -29,12 +29,14 @@ #define _PRIVATE_BIONIC_GLOBALS_H #include +#include "private/bionic_malloc_dispatch.h" #include "private/bionic_vdso.h" #include "private/WriteProtected.h" struct libc_globals { vdso_entry vdso[VDSO_END]; long setjmp_cookie; + MallocDebug malloc_dispatch; }; __LIBC_HIDDEN__ extern WriteProtected __libc_globals; @@ -44,5 +46,5 @@ __LIBC_HIDDEN__ void __libc_init_vdso(libc_globals* globals, KernelArgumentBlock& args); __LIBC_HIDDEN__ void __libc_init_setjmp_cookie(libc_globals* globals, KernelArgumentBlock& args); - +__LIBC_HIDDEN__ void __libc_init_malloc(libc_globals* globals); #endif diff --git a/libc/private/bionic_malloc_dispatch.h b/libc/private/bionic_malloc_dispatch.h new file mode 100644 index 000000000..34fb8983a --- /dev/null +++ b/libc/private/bionic_malloc_dispatch.h @@ -0,0 +1,68 @@ +/* + * Copyright (C) 2015 The Android Open Source Project + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS + * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT + * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef _PRIVATE_BIONIC_MALLOC_DISPATCH_H +#define _PRIVATE_BIONIC_MALLOC_DISPATCH_H + +#include +#include "private/bionic_config.h" + +/* Entry in malloc dispatch table. */ +typedef void* (*MallocDebugCalloc)(size_t, size_t); +typedef void (*MallocDebugFree)(void*); +typedef struct mallinfo (*MallocDebugMallinfo)(); +typedef void* (*MallocDebugMalloc)(size_t); +typedef size_t (*MallocDebugMallocUsableSize)(const void*); +typedef void* (*MallocDebugMemalign)(size_t, size_t); +typedef int (*MallocDebugPosixMemalign)(void**, size_t, size_t); +#if defined(HAVE_DEPRECATED_MALLOC_FUNCS) +typedef void* (*MallocDebugPvalloc)(size_t); +#endif +typedef void* (*MallocDebugRealloc)(void*, size_t); +#if defined(HAVE_DEPRECATED_MALLOC_FUNCS) +typedef void* (*MallocDebugValloc)(size_t); +#endif + +struct MallocDebug { + MallocDebugCalloc calloc; + MallocDebugFree free; + MallocDebugMallinfo mallinfo; + MallocDebugMalloc malloc; + MallocDebugMallocUsableSize malloc_usable_size; + MallocDebugMemalign memalign; + MallocDebugPosixMemalign posix_memalign; +#if defined(HAVE_DEPRECATED_MALLOC_FUNCS) + MallocDebugPvalloc pvalloc; +#endif + MallocDebugRealloc realloc; +#if defined(HAVE_DEPRECATED_MALLOC_FUNCS) + MallocDebugValloc valloc; +#endif +} __attribute__((aligned(32))); + +#endif