android_system_core/llkd/tests/llkd_test.cpp

244 lines
8.1 KiB
C++
Raw Normal View History

llkd: add live-lock daemon Introduce a standalone live-lock daemon (llkd), to catch kernel or native user space deadlocks and take mitigating actions. Will also configure [khungtaskd] to fortify the actions. If a thread is in D or Z state with no forward progress for longer than ro.llk.timeout_ms, or ro.llk.[D|Z].timeout_ms, kill the process or parent process respectively. If another scan shows the same process continues to exist, then have a confirmed live-lock condition and need to panic. Panic the kernel in a manner to provide the greatest bugreporting details as to the condition. Add a alarm self watchdog should llkd ever get locked up that is double the expected time to flow through the mainloop. Sampling is every ro.llk_sample_ms. Default will not monitor init, or [kthreadd] and all that [kthreadd] spawns. This reduces the effectiveness of llkd by limiting its coverage. If in the future, if value in covering kthreadd spawned threads, the requirement will be to code drivers so that they do not remain in a persistent 'D' state, or that they have mechanisms to recover the thread should it be killed externally. Then the blacklists can be adjusted accordingly if these conditions are met. An accompanying gTest set have been added, and will setup a persistent D or Z process, with and without forward progress, but not in a live-lock state because that would require a buggy kernel, or a module or kernel modification to stimulate. Android Properties llkd respond to (*_ms parms are in milliseconds): - ro.config.low_ram default false, if true do not sysrq t (dump all threads). - ro.llk.enable default false, allow live-lock daemon to be enabled. - ro.khungtask.enable default false, allow [khungtaskd] to be enabled. - ro.llk.mlockall default true, allow mlock'd live-lock daemon. - ro.khungtask.timeout default 12 minutes. - ro.llk.timeout_ms default 10 minutes, D or Z maximum timelimit, double this value and it sets the alarm watchdog for llkd. - ro.llk.D.timeout_ms default ro.llk.timeout_ms, D maximum timelimit. - ro.llk.Z.timeout_ms default ro.llk.timeout_ms, Z maximum timelimit. - ro.llk.check_ms default 2 minutes sampling interval (ro.llk.timeout_ms / 5) for threads in D or Z state. - ro.llk.blacklist.process default 0,1,2 (kernel, init and [kthreadd]), and process names (/comm or /cmdline) init,[kthreadd], lmkd,lmkd.llkd,llkd,[khungtaskd],watchdogd,[watchdogd], [watchdogd/0] ... - ro.llk.blacklist.parent default 0,2 (kernel and [kthreadd]) and "[kthreadd]". A comma separated lists of process ids, /comm names or /cmdline names. - ro.llk.blacklist.uid default <empty>, comma separated list of uid numbers or names from getpwuid/getpwnam. Test: llkd_unit_test Bug: 33808187 Bug: 72838192 Change-Id: I32e8aa78aef10834e093265d0f3ed5b4199807c6
2018-02-20 18:47:40 +00:00
/*
* Copyright (C) 2018 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include <signal.h>
#include <stdint.h>
#include <sys/types.h>
#include <sys/wait.h>
#include <unistd.h>
#include <chrono>
#include <iostream>
#include <string>
#include <android-base/properties.h>
#include <gtest/gtest.h>
#include <log/log_time.h> // for MS_PER_SEC and US_PER_SEC
#include "llkd.h"
using namespace std::chrono;
using namespace std::chrono_literals;
namespace {
milliseconds GetUintProperty(const std::string& key, milliseconds def) {
return milliseconds(android::base::GetUintProperty(key, static_cast<uint64_t>(def.count()),
static_cast<uint64_t>(def.max().count())));
}
seconds GetUintProperty(const std::string& key, seconds def) {
return seconds(android::base::GetUintProperty(key, static_cast<uint64_t>(def.count()),
static_cast<uint64_t>(def.max().count())));
}
// GTEST_LOG_(WARNING) output is fugly, this has much less noise
// ToDo: look into fixing googletest to produce output that matches style of
// all the other status messages, and can switch off __line__ and
// __function__ noise
#define GTEST_LOG_WARNING std::cerr << "[ WARNING ] "
#define GTEST_LOG_INFO std::cerr << "[ INFO ] "
// Properties is _not_ a high performance ABI!
void rest() {
usleep(200000);
}
void execute(const char* command) {
if (getuid() || system(command)) {
system((std::string("su root ") + command).c_str());
}
}
seconds llkdSleepPeriod(char state) {
auto default_enable = android::base::GetBoolProperty(LLK_ENABLE_PROPERTY, LLK_ENABLE_DEFAULT);
if (android::base::GetProperty(LLK_ENABLE_PROPERTY, "nothing") == "nothing") {
GTEST_LOG_INFO << LLK_ENABLE_PROPERTY " defaults to " << (default_enable ? "true" : "false")
<< "\n";
}
// Hail Mary hope is unconfigured.
if ((GetUintProperty(LLK_TIMEOUT_MS_PROPERTY, LLK_TIMEOUT_MS_DEFAULT) !=
duration_cast<milliseconds>(120s)) ||
(GetUintProperty(LLK_CHECK_MS_PROPERTY,
LLK_TIMEOUT_MS_DEFAULT / LLK_CHECKS_PER_TIMEOUT_DEFAULT) !=
duration_cast<milliseconds>(10s))) {
execute("stop llkd");
rest();
std::string setprop("setprop ");
execute((setprop + LLK_TIMEOUT_MS_PROPERTY + " 120000").c_str());
rest();
execute((setprop + KHT_TIMEOUT_PROPERTY + " 130").c_str());
rest();
execute((setprop + LLK_CHECK_MS_PROPERTY + " 10000").c_str());
rest();
execute((setprop + LLK_ENABLE_PROPERTY + " true").c_str());
rest();
}
default_enable = android::base::GetBoolProperty(LLK_ENABLE_PROPERTY, false);
if (default_enable) {
execute("start llkd");
rest();
GTEST_LOG_INFO << "llkd enabled\n";
} else {
GTEST_LOG_WARNING << "llkd disabled\n";
}
/* KISS follows llk_init() */
milliseconds llkTimeoutMs = LLK_TIMEOUT_MS_DEFAULT;
seconds khtTimeout = duration_cast<seconds>(
llkTimeoutMs * (1 + LLK_CHECKS_PER_TIMEOUT_DEFAULT) / LLK_CHECKS_PER_TIMEOUT_DEFAULT);
khtTimeout = GetUintProperty(KHT_TIMEOUT_PROPERTY, khtTimeout);
llkTimeoutMs =
khtTimeout * LLK_CHECKS_PER_TIMEOUT_DEFAULT / (1 + LLK_CHECKS_PER_TIMEOUT_DEFAULT);
llkTimeoutMs = GetUintProperty(LLK_TIMEOUT_MS_PROPERTY, llkTimeoutMs);
if (llkTimeoutMs < LLK_TIMEOUT_MS_MINIMUM) {
llkTimeoutMs = LLK_TIMEOUT_MS_MINIMUM;
}
milliseconds llkCheckMs = llkTimeoutMs / LLK_CHECKS_PER_TIMEOUT_DEFAULT;
auto timeout = GetUintProperty(
(state == 'Z') ? LLK_Z_TIMEOUT_MS_PROPERTY : LLK_D_TIMEOUT_MS_PROPERTY, llkTimeoutMs);
if (timeout < LLK_TIMEOUT_MS_MINIMUM) {
timeout = LLK_TIMEOUT_MS_MINIMUM;
}
if (llkCheckMs > timeout) {
llkCheckMs = timeout;
}
llkCheckMs = GetUintProperty(LLK_CHECK_MS_PROPERTY, llkCheckMs);
timeout += llkCheckMs;
auto sec = duration_cast<seconds>(timeout);
if (sec == 0s) {
++sec;
} else if (sec > 59s) {
GTEST_LOG_WARNING << "llkd is configured for about " << duration_cast<minutes>(sec).count()
<< " minutes to react\n";
}
// 33% margin for the test to naturally timeout waiting for llkd to respond
return (sec * 4 + 2s) / 3;
}
inline void waitForPid(pid_t child_pid) {
int wstatus;
ASSERT_LE(0, waitpid(child_pid, &wstatus, 0));
EXPECT_FALSE(WIFEXITED(wstatus)) << "[ INFO ] exit=" << WEXITSTATUS(wstatus);
ASSERT_TRUE(WIFSIGNALED(wstatus));
ASSERT_EQ(WTERMSIG(wstatus), SIGKILL);
}
} // namespace
// The tests that use this helper are to simulate processes stuck in 'D'
// state that are experiencing forward scheduled progress. As such the
// expectation is that llkd will _not_ perform any mitigations. The sleepfor
// argument helps us set the amount of forward scheduler progress.
static void llkd_driver_ABA(const microseconds sleepfor) {
const auto period = llkdSleepPeriod('D');
if (period <= sleepfor) {
GTEST_LOG_WARNING << "llkd configuration too short for "
<< duration_cast<milliseconds>(sleepfor).count() << "ms work cycle\n";
return;
}
auto child_pid = fork();
ASSERT_LE(0, child_pid);
int wstatus;
if (!child_pid) {
auto ratio = period / sleepfor;
ASSERT_LT(0, ratio);
// vfork() parent is uninterruptable D state waiting for child to exec()
while (--ratio > 0) {
auto driver_pid = vfork();
ASSERT_LE(0, driver_pid);
if (driver_pid) { // parent
waitpid(driver_pid, &wstatus, 0);
if (!WIFEXITED(wstatus)) {
exit(42);
}
if (WEXITSTATUS(wstatus) != 42) {
exit(42);
}
} else {
usleep(sleepfor.count());
exit(42);
}
}
exit(0);
}
ASSERT_LE(0, waitpid(child_pid, &wstatus, 0));
EXPECT_TRUE(WIFEXITED(wstatus));
if (WIFEXITED(wstatus)) {
EXPECT_EQ(0, WEXITSTATUS(wstatus));
}
ASSERT_FALSE(WIFSIGNALED(wstatus)) << "[ INFO ] signo=" << WTERMSIG(wstatus);
}
TEST(llkd, driver_ABA_fast) {
llkd_driver_ABA(5ms);
}
TEST(llkd, driver_ABA_slow) {
llkd_driver_ABA(1s);
}
TEST(llkd, driver_ABA_glacial) {
llkd_driver_ABA(1min);
}
// Following tests must be last in this file to capture possible errant
// kernel_panic mitigation failure.
// The following tests simulate processes stick in 'Z' or 'D' state with
// no forward scheduling progress, but interruptible. As such the expectation
// is that llkd will perform kill mitigation and not progress to kernel_panic.
TEST(llkd, zombie) {
const auto period = llkdSleepPeriod('Z');
/* Create a Persistent Zombie Process */
pid_t child_pid = fork();
ASSERT_LE(0, child_pid);
if (!child_pid) {
auto zombie_pid = fork();
ASSERT_LE(0, zombie_pid);
if (!zombie_pid) {
sleep(1);
exit(0);
}
sleep(period.count());
exit(42);
}
waitForPid(child_pid);
}
TEST(llkd, driver) {
const auto period = llkdSleepPeriod('D');
/* Create a Persistent Device Process */
auto child_pid = fork();
ASSERT_LE(0, child_pid);
if (!child_pid) {
// vfork() parent is uninterruptable D state waiting for child to exec()
auto driver_pid = vfork();
ASSERT_LE(0, driver_pid);
sleep(period.count());
exit(driver_pid ? 42 : 0);
}
waitForPid(child_pid);
}