Eliminate deadlock in forked child due to delayed resetting mutex lock

For some program implementation, the pattern like below, calling 
pthread_atfork to register atfork interfaces.

    pthread_atfork(&atfork_prepare, &atfork_parent, &atfork_child);

When the program is expected to reopen the shared library's handle 
inherited from parent in child process. Maybe, dlclose is called in 
atfork_child to release the shared library handle before reopen it. 
Then, dlclose will indrectly call _cxa_finalize and finaly call 
__unregister_atfork when dso is not NULL.

    atfork_child() -> dlclose() -> __on_dlclose() 
    	-> __cxa_finalize() -> __unregister_atfork(dso)

In __unregister_atfork, firstly, it try to hold the g_atfork_list_mutex 
lock to operate the g_atfork_list. Due to the registered atfork_child is
executed before resetting g_atfork_list_mutex lock in child, the child 
process will be blocked here because of deadlock.

Test: bionic-unit-tests32 --gtest_filter=pthread.pthread_atfork_child_with_dlclose 
without the fixing, the test will be timeout.

Change-Id: I35d3001682c836e0955d6d681bc5f9297fad0c7b
Signed-off-by: Mingwei Shi <mingwei.shi@intel.com>
Signed-off-by: Qiming Shi <qiming.shi@intel.com>
Signed-off-by: Chao Xie <chao.xie@intel.com>

libc/bionic/pthread_atfork.cpp

@@ -130,13 +130,15 @@ void __bionic_atfork_run_prepare() {
 }
 
 void __bionic_atfork_run_child() {
+  g_atfork_list_mutex = PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP;
+
+  pthread_mutex_lock(&g_atfork_list_mutex);
   g_atfork_list.walk_forward([](atfork_t* it) {
     if (it->child != nullptr) {
       it->child();
     }
   });
-
-  g_atfork_list_mutex = PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP;
+  pthread_mutex_unlock(&g_atfork_list_mutex);
 }
 
 void __bionic_atfork_run_parent() {

tests/pthread_dlfcn_test.cpp

@@ -37,6 +37,11 @@ static void AtForkChild2() { g_atfork_child_calls = (g_atfork_child_calls * 10)
 static void AtForkChild3() { g_atfork_child_calls = (g_atfork_child_calls * 10) + 3; }
 static void AtForkChild4() { g_atfork_child_calls = (g_atfork_child_calls * 10) + 4; }
 
+static void* g_atfork_test_handle = nullptr;
+static void AtForkPrepare() {}
+static void AtForkParent() {}
+static void AtForkChild() { dlclose(g_atfork_test_handle); g_atfork_test_handle = dlopen("libtest_pthread_atfork.so", RTLD_NOW | RTLD_LOCAL); }
+
 TEST(pthread, pthread_atfork_with_dlclose) {
   ASSERT_EQ(0, pthread_atfork(AtForkPrepare1, AtForkParent1, AtForkChild1));
 
@@ -82,3 +87,28 @@ TEST(pthread, pthread_atfork_with_dlclose) {
 
   AssertChildExited(pid, 0);
 }
+
+TEST(pthread, pthread_atfork_child_with_dlclose) {
+
+  g_atfork_test_handle = dlopen("libtest_pthread_atfork.so", RTLD_NOW | RTLD_LOCAL);
+  ASSERT_TRUE(g_atfork_test_handle != nullptr) << dlerror();
+  typedef int (*fn_t)(void (*)(void), void (*)(void), void (*)(void));
+  fn_t fn = reinterpret_cast<fn_t>(dlsym(g_atfork_test_handle, "proxy_pthread_atfork"));
+  ASSERT_TRUE(fn != nullptr) << dlerror();
+  // the library registers 2 additional atfork handlers in a constructor
+
+  ASSERT_EQ(0, pthread_atfork(AtForkPrepare, AtForkParent, AtForkChild));
+
+  pid_t pid = fork();
+
+  ASSERT_NE(-1, pid) << strerror(errno);
+
+  if (pid == 0) {
+    _exit(0);
+  }
+
+  AssertChildExited(pid, 0);
+
+  EXPECT_EQ(0, dlclose(g_atfork_test_handle));
+  g_atfork_test_handle = nullptr;
+}