Add MetricsProcessor to calculate metrics stats
This CL adds MetricsProcessor class to calculate metrics stats for both device tests and host tests. Test: atest MicrodroidBenchmarks Change-Id: I4f742d6800f14c857a5d0e0718e09870e8ca6547
This commit is contained in:
Alice Wang
parent
6f5ff860b1
commit
bc039c099b
|
|
@ -10,6 +10,7 @@ android_test {
|
|||
srcs: ["src/java/**/*.java"],
|
||||
static_libs: [
|
||||
"MicrodroidDeviceTestHelper",
|
||||
"MicrodroidTestHelper",
|
||||
"androidx.test.runner",
|
||||
"androidx.test.ext.junit",
|
||||
"com.android.microdroid.testservice-java",
|
||||
|
|
|
|||
|
|
@ -32,6 +32,7 @@ import android.system.virtualmachine.VirtualMachineException;
|
|||
import android.util.Log;
|
||||
|
||||
import com.android.microdroid.test.MicrodroidDeviceTestBase;
|
||||
import com.android.microdroid.test.common.MetricsProcessor;
|
||||
import com.android.microdroid.testservice.IBenchmarkService;
|
||||
|
||||
import org.junit.Before;
|
||||
|
|
@ -46,6 +47,7 @@ import java.io.IOException;
|
|||
import java.nio.file.Files;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.atomic.AtomicReference;
|
||||
|
||||
@RunWith(Parameterized.class)
|
||||
|
|
@ -68,6 +70,8 @@ public class MicrodroidBenchmarks extends MicrodroidDeviceTestBase {
|
|||
|
||||
@Parameterized.Parameter public boolean mProtectedVm;
|
||||
|
||||
private final MetricsProcessor mMetricsProcessor = new MetricsProcessor(METRIC_NAME_PREFIX);
|
||||
|
||||
private Instrumentation mInstrumentation;
|
||||
|
||||
@Before
|
||||
|
|
@ -153,11 +157,11 @@ public class MicrodroidBenchmarks extends MicrodroidDeviceTestBase {
|
|||
userspaceBootTimeMetrics.add(result.getUserspaceElapsedNanoTime() / nanoToMilli);
|
||||
}
|
||||
|
||||
reportMetrics(vmStartingTimeMetrics, "vm_starting_time_", "_ms");
|
||||
reportMetrics(bootTimeMetrics, "boot_time_", "_ms");
|
||||
reportMetrics(bootloaderTimeMetrics, "bootloader_time_", "_ms");
|
||||
reportMetrics(kernelBootTimeMetrics, "kernel_boot_time_", "_ms");
|
||||
reportMetrics(userspaceBootTimeMetrics, "userspace_boot_time_", "_ms");
|
||||
reportMetrics(vmStartingTimeMetrics, "vm_starting_time", "ms");
|
||||
reportMetrics(bootTimeMetrics, "boot_time", "ms");
|
||||
reportMetrics(bootloaderTimeMetrics, "bootloader_time", "ms");
|
||||
reportMetrics(kernelBootTimeMetrics, "kernel_boot_time", "ms");
|
||||
reportMetrics(userspaceBootTimeMetrics, "userspace_boot_time", "ms");
|
||||
}
|
||||
|
||||
@Test
|
||||
|
|
@ -187,7 +191,7 @@ public class MicrodroidBenchmarks extends MicrodroidDeviceTestBase {
|
|||
mInner.newVmConfigBuilder("assets/vm_config_io.json")
|
||||
.debugLevel(DebugLevel.FULL)
|
||||
.build();
|
||||
List<Double> transferRates = new ArrayList<>();
|
||||
List<Double> transferRates = new ArrayList<>(IO_TEST_TRIAL_COUNT);
|
||||
|
||||
for (int i = 0; i < IO_TEST_TRIAL_COUNT; ++i) {
|
||||
int port = (mProtectedVm ? 5666 : 6666) + i;
|
||||
|
|
@ -196,7 +200,7 @@ public class MicrodroidBenchmarks extends MicrodroidDeviceTestBase {
|
|||
VirtualMachine vm = mInner.getVirtualMachineManager().get(vmName);
|
||||
BenchmarkVmListener.create(new VsockListener(transferRates, port)).runToFinish(TAG, vm);
|
||||
}
|
||||
reportMetrics(transferRates, "vsock/transfer_host_to_vm_", "_mb_per_sec");
|
||||
reportMetrics(transferRates, "vsock/transfer_host_to_vm", "mb_per_sec");
|
||||
}
|
||||
|
||||
@Test
|
||||
|
|
@ -214,7 +218,7 @@ public class MicrodroidBenchmarks extends MicrodroidDeviceTestBase {
|
|||
mInner.newVmConfigBuilder("assets/vm_config_io.json")
|
||||
.debugLevel(DebugLevel.FULL)
|
||||
.build();
|
||||
List<Double> readRates = new ArrayList<>();
|
||||
List<Double> readRates = new ArrayList<>(IO_TEST_TRIAL_COUNT);
|
||||
|
||||
for (int i = 0; i < IO_TEST_TRIAL_COUNT + 1; ++i) {
|
||||
if (i == 1) {
|
||||
|
|
@ -230,33 +234,15 @@ public class MicrodroidBenchmarks extends MicrodroidDeviceTestBase {
|
|||
.runToFinish(TAG, vm);
|
||||
}
|
||||
reportMetrics(
|
||||
readRates,
|
||||
isRand ? "virtio-blk/rand_read_" : "virtio-blk/seq_read_",
|
||||
"_mb_per_sec");
|
||||
readRates, isRand ? "virtio-blk/rand_read" : "virtio-blk/seq_read", "mb_per_sec");
|
||||
}
|
||||
|
||||
private void reportMetrics(List<Double> data, String base, String suffix) {
|
||||
double sum = 0;
|
||||
double min = Double.MAX_VALUE;
|
||||
double max = Double.MIN_VALUE;
|
||||
for (double d : data) {
|
||||
sum += d;
|
||||
if (min > d) min = d;
|
||||
if (max < d) max = d;
|
||||
}
|
||||
double avg = sum / data.size();
|
||||
double sqSum = 0;
|
||||
for (double d : data) {
|
||||
sqSum += (d - avg) * (d - avg);
|
||||
}
|
||||
double stdDev = Math.sqrt(sqSum / (data.size() - 1));
|
||||
|
||||
private void reportMetrics(List<Double> metrics, String name, String unit) {
|
||||
Map<String, Double> stats = mMetricsProcessor.computeStats(metrics, name, unit);
|
||||
Bundle bundle = new Bundle();
|
||||
String prefix = METRIC_NAME_PREFIX + base;
|
||||
bundle.putDouble(prefix + "min" + suffix, min);
|
||||
bundle.putDouble(prefix + "max" + suffix, max);
|
||||
bundle.putDouble(prefix + "average" + suffix, avg);
|
||||
bundle.putDouble(prefix + "stdev" + suffix, stdDev);
|
||||
for (Map.Entry<String, Double> entry : stats.entrySet()) {
|
||||
bundle.putDouble(entry.getKey(), entry.getValue());
|
||||
}
|
||||
mInstrumentation.sendStatus(0, bundle);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ java_test_host {
|
|||
],
|
||||
static_libs: [
|
||||
"MicrodroidHostTestHelper",
|
||||
"MicrodroidTestHelper",
|
||||
],
|
||||
test_suites: [
|
||||
"general-tests",
|
||||
|
|
|
|||
|
|
@ -16,8 +16,6 @@
|
|||
|
||||
package android.avf.test;
|
||||
|
||||
import android.platform.test.annotations.RootPermissionTest;
|
||||
|
||||
import static com.android.tradefed.testtype.DeviceJUnit4ClassRunner.TestMetrics;
|
||||
|
||||
import static com.google.common.truth.Truth.assertWithMessage;
|
||||
|
|
@ -25,8 +23,11 @@ import static com.google.common.truth.TruthJUnit.assume;
|
|||
|
||||
import static org.junit.Assume.assumeTrue;
|
||||
|
||||
import android.platform.test.annotations.RootPermissionTest;
|
||||
|
||||
import com.android.microdroid.test.CommandRunner;
|
||||
import com.android.microdroid.test.MicrodroidHostTestCaseBase;
|
||||
import com.android.microdroid.test.common.MetricsProcessor;
|
||||
import com.android.tradefed.log.LogUtil.CLog;
|
||||
import com.android.tradefed.testtype.DeviceJUnit4ClassRunner;
|
||||
import com.android.tradefed.util.CommandResult;
|
||||
|
|
@ -36,6 +37,9 @@ import org.junit.Before;
|
|||
import org.junit.Test;
|
||||
import org.junit.runner.RunWith;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
|
|
@ -61,6 +65,8 @@ public final class AVFHostTestCase extends MicrodroidHostTestCaseBase {
|
|||
private static final int ROUND_COUNT = 5;
|
||||
private static final String METRIC_PREFIX = "avf_perf/hostside/";
|
||||
|
||||
private final MetricsProcessor mMetricsProcessor = new MetricsProcessor(METRIC_PREFIX);
|
||||
|
||||
@Before
|
||||
public void setUp() throws Exception {
|
||||
testIfDeviceIsCapable(getDevice());
|
||||
|
|
@ -80,11 +86,10 @@ public final class AVFHostTestCase extends MicrodroidHostTestCaseBase {
|
|||
|
||||
@Test
|
||||
public void testBootEnableAndDisablePKVM() throws Exception {
|
||||
|
||||
testPKVMStatusSwitchSupported();
|
||||
|
||||
double[] bootWithPKVMEnableTime = new double[ROUND_COUNT];
|
||||
double[] bootWithoutPKVMEnableTime = new double[ROUND_COUNT];
|
||||
List<Double> bootWithPKVMEnableTime = new ArrayList<>(ROUND_COUNT);
|
||||
List<Double> bootWithoutPKVMEnableTime = new ArrayList<>(ROUND_COUNT);
|
||||
|
||||
for (int round = 0; round < ROUND_COUNT; ++round) {
|
||||
|
||||
|
|
@ -93,7 +98,7 @@ public final class AVFHostTestCase extends MicrodroidHostTestCaseBase {
|
|||
rebootFromBootloaderAndWaitBootCompleted();
|
||||
long elapsedWithPKVMEnable = System.nanoTime() - start;
|
||||
double elapsedSec = elapsedWithPKVMEnable / NANOS_IN_SEC;
|
||||
bootWithPKVMEnableTime[round] = elapsedSec;
|
||||
bootWithPKVMEnableTime.add(elapsedSec);
|
||||
CLog.i("Boot time with PKVM enable took " + elapsedSec + "s");
|
||||
|
||||
setPKVMStatusWithRebootToBootloader(false);
|
||||
|
|
@ -101,20 +106,20 @@ public final class AVFHostTestCase extends MicrodroidHostTestCaseBase {
|
|||
rebootFromBootloaderAndWaitBootCompleted();
|
||||
long elapsedWithoutPKVMEnable = System.nanoTime() - start;
|
||||
elapsedSec = elapsedWithoutPKVMEnable / NANOS_IN_SEC;
|
||||
bootWithoutPKVMEnableTime[round] = elapsedSec;
|
||||
bootWithoutPKVMEnableTime.add(elapsedSec);
|
||||
CLog.i("Boot time with PKVM disable took " + elapsedSec + "s");
|
||||
}
|
||||
|
||||
reportMetric("boot_time_with_pkvm_enable", "s", bootWithPKVMEnableTime);
|
||||
reportMetric("boot_time_with_pkvm_disable", "s", bootWithoutPKVMEnableTime);
|
||||
reportMetric(bootWithPKVMEnableTime, "boot_time_with_pkvm_enable", "s");
|
||||
reportMetric(bootWithoutPKVMEnableTime, "boot_time_with_pkvm_disable", "s");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testBootWithAndWithoutCompOS() throws Exception {
|
||||
assume().withMessage("Skip on CF; too slow").that(isCuttlefish()).isFalse();
|
||||
|
||||
double[] bootWithCompOsTime = new double[ROUND_COUNT];
|
||||
double[] bootWithoutCompOsTime = new double[ROUND_COUNT];
|
||||
List<Double> bootWithCompOsTime = new ArrayList<>(ROUND_COUNT);
|
||||
List<Double> bootWithoutCompOsTime = new ArrayList<>(ROUND_COUNT);
|
||||
|
||||
for (int round = 0; round < ROUND_COUNT; ++round) {
|
||||
|
||||
|
|
@ -125,7 +130,7 @@ public final class AVFHostTestCase extends MicrodroidHostTestCaseBase {
|
|||
rebootAndWaitBootCompleted();
|
||||
long elapsedWithCompOS = System.nanoTime() - start;
|
||||
double elapsedSec = elapsedWithCompOS / NANOS_IN_SEC;
|
||||
bootWithCompOsTime[round] = elapsedSec;
|
||||
bootWithCompOsTime.add(elapsedSec);
|
||||
CLog.i("Boot time with compilation OS took " + elapsedSec + "s");
|
||||
|
||||
// Boot time without compilation OS test.
|
||||
|
|
@ -134,12 +139,12 @@ public final class AVFHostTestCase extends MicrodroidHostTestCaseBase {
|
|||
rebootAndWaitBootCompleted();
|
||||
long elapsedWithoutCompOS = System.nanoTime() - start;
|
||||
elapsedSec = elapsedWithoutCompOS / NANOS_IN_SEC;
|
||||
bootWithoutCompOsTime[round] = elapsedSec;
|
||||
bootWithoutCompOsTime.add(elapsedSec);
|
||||
CLog.i("Boot time without compilation OS took " + elapsedSec + "s");
|
||||
}
|
||||
|
||||
reportMetric("boot_time_with_compos", "s", bootWithCompOsTime);
|
||||
reportMetric("boot_time_without_compos", "s", bootWithoutCompOsTime);
|
||||
reportMetric(bootWithCompOsTime, "boot_time_with_compos", "s");
|
||||
reportMetric(bootWithoutCompOsTime, "boot_time_without_compos", "s");
|
||||
}
|
||||
|
||||
private void testPKVMStatusSwitchSupported() throws Exception {
|
||||
|
|
@ -154,31 +159,12 @@ public final class AVFHostTestCase extends MicrodroidHostTestCaseBase {
|
|||
assumeTrue(!result.getStderr().contains("Invalid oem command"));
|
||||
}
|
||||
|
||||
private void reportMetric(String name, String unit, double[] values) {
|
||||
double sum = 0;
|
||||
double min = Double.MAX_VALUE;
|
||||
double max = Double.MIN_VALUE;
|
||||
|
||||
for (double val : values) {
|
||||
sum += val;
|
||||
min = val < min ? val : min;
|
||||
max = val > max ? val : max;
|
||||
}
|
||||
|
||||
double average = sum / values.length;
|
||||
|
||||
double variance = 0;
|
||||
for (double val : values) {
|
||||
final double tmp = val - average;
|
||||
variance += tmp * tmp;
|
||||
}
|
||||
double stdev = Math.sqrt(variance / (double) (values.length - 1));
|
||||
|
||||
private void reportMetric(List<Double> data, String name, String unit) {
|
||||
Map<String, Double> stats = mMetricsProcessor.computeStats(data, name, unit);
|
||||
TestMetrics metrics = new TestMetrics();
|
||||
metrics.addTestMetric(METRIC_PREFIX + name + "_average_" + unit, Double.toString(average));
|
||||
metrics.addTestMetric(METRIC_PREFIX + name + "_min_" + unit, Double.toString(min));
|
||||
metrics.addTestMetric(METRIC_PREFIX + name + "_max_" + unit, Double.toString(max));
|
||||
metrics.addTestMetric(METRIC_PREFIX + name + "_stdev_" + unit, Double.toString(stdev));
|
||||
for (Map.Entry<String, Double> entry : stats.entrySet()) {
|
||||
metrics.addTestMetric(entry.getKey(), Double.toString(entry.getValue()));
|
||||
}
|
||||
}
|
||||
|
||||
private void setPKVMStatusWithRebootToBootloader(boolean isEnable) throws Exception {
|
||||
|
|
|
|||
|
|
@ -8,9 +8,15 @@ java_library_static {
|
|||
host_supported: true,
|
||||
}
|
||||
|
||||
java_library_static {
|
||||
name: "MicrodroidTestHelper",
|
||||
srcs: ["src/java/com/android/microdroid/test/common/*.java"],
|
||||
host_supported: true,
|
||||
}
|
||||
|
||||
java_library_static {
|
||||
name: "MicrodroidDeviceTestHelper",
|
||||
srcs: ["src/java/com/android/microdroid/**/*.java"],
|
||||
srcs: ["src/java/com/android/microdroid/test/*.java"],
|
||||
static_libs: [
|
||||
"androidx.test.runner",
|
||||
"androidx.test.ext.junit",
|
||||
|
|
|
|||
|
|
@ -0,0 +1,60 @@
|
|||
/*
|
||||
* Copyright (C) 2022 The Android Open Source Project
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.android.microdroid.test.common;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
/** This class processes the metrics for both device tests and host tests. */
|
||||
public final class MetricsProcessor {
|
||||
private final String mPrefix;
|
||||
|
||||
public MetricsProcessor(String prefix) {
|
||||
mPrefix = prefix;
|
||||
}
|
||||
|
||||
/**
|
||||
* Computes the min, max, average and standard deviation of the given metrics and saves them in
|
||||
* a {@link Map} with the corresponding keys equal to [mPrefix + name +
|
||||
* _[min|max|average|stdev]_ + unit].
|
||||
*/
|
||||
public Map<String, Double> computeStats(List<Double> metrics, String name, String unit) {
|
||||
double sum = 0;
|
||||
double min = Double.MAX_VALUE;
|
||||
double max = Double.MIN_VALUE;
|
||||
for (double d : metrics) {
|
||||
sum += d;
|
||||
if (min > d) min = d;
|
||||
if (max < d) max = d;
|
||||
}
|
||||
double avg = sum / metrics.size();
|
||||
double sqSum = 0;
|
||||
for (double d : metrics) {
|
||||
sqSum += (d - avg) * (d - avg);
|
||||
}
|
||||
double stdDev = Math.sqrt(sqSum / (metrics.size() - 1));
|
||||
|
||||
Map<String, Double> stats = new HashMap<String, Double>();
|
||||
String prefix = mPrefix + name;
|
||||
stats.put(prefix + "_min_" + unit, min);
|
||||
stats.put(prefix + "_max_" + unit, max);
|
||||
stats.put(prefix + "_average_" + unit, avg);
|
||||
stats.put(prefix + "_stdev_" + unit, stdDev);
|
||||
return stats;
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue