diff --git a/tests/benchmark/Android.bp b/tests/benchmark/Android.bp index 0d1cd91c..bf78202c 100644 --- a/tests/benchmark/Android.bp +++ b/tests/benchmark/Android.bp @@ -10,6 +10,7 @@ android_test { srcs: ["src/java/**/*.java"], static_libs: [ "MicrodroidDeviceTestHelper", + "MicrodroidTestHelper", "androidx.test.runner", "androidx.test.ext.junit", "com.android.microdroid.testservice-java", diff --git a/tests/benchmark/src/java/com/android/microdroid/benchmark/MicrodroidBenchmarks.java b/tests/benchmark/src/java/com/android/microdroid/benchmark/MicrodroidBenchmarks.java index 200b7bfe..8e65132b 100644 --- a/tests/benchmark/src/java/com/android/microdroid/benchmark/MicrodroidBenchmarks.java +++ b/tests/benchmark/src/java/com/android/microdroid/benchmark/MicrodroidBenchmarks.java @@ -32,6 +32,7 @@ import android.system.virtualmachine.VirtualMachineException; import android.util.Log; import com.android.microdroid.test.MicrodroidDeviceTestBase; +import com.android.microdroid.test.common.MetricsProcessor; import com.android.microdroid.testservice.IBenchmarkService; import org.junit.Before; @@ -46,6 +47,7 @@ import java.io.IOException; import java.nio.file.Files; import java.util.ArrayList; import java.util.List; +import java.util.Map; import java.util.concurrent.atomic.AtomicReference; @RunWith(Parameterized.class) @@ -68,6 +70,8 @@ public class MicrodroidBenchmarks extends MicrodroidDeviceTestBase { @Parameterized.Parameter public boolean mProtectedVm; + private final MetricsProcessor mMetricsProcessor = new MetricsProcessor(METRIC_NAME_PREFIX); + private Instrumentation mInstrumentation; @Before @@ -153,11 +157,11 @@ public class MicrodroidBenchmarks extends MicrodroidDeviceTestBase { userspaceBootTimeMetrics.add(result.getUserspaceElapsedNanoTime() / nanoToMilli); } - reportMetrics(vmStartingTimeMetrics, "vm_starting_time_", "_ms"); - reportMetrics(bootTimeMetrics, "boot_time_", "_ms"); - reportMetrics(bootloaderTimeMetrics, "bootloader_time_", "_ms"); - reportMetrics(kernelBootTimeMetrics, "kernel_boot_time_", "_ms"); - reportMetrics(userspaceBootTimeMetrics, "userspace_boot_time_", "_ms"); + reportMetrics(vmStartingTimeMetrics, "vm_starting_time", "ms"); + reportMetrics(bootTimeMetrics, "boot_time", "ms"); + reportMetrics(bootloaderTimeMetrics, "bootloader_time", "ms"); + reportMetrics(kernelBootTimeMetrics, "kernel_boot_time", "ms"); + reportMetrics(userspaceBootTimeMetrics, "userspace_boot_time", "ms"); } @Test @@ -187,7 +191,7 @@ public class MicrodroidBenchmarks extends MicrodroidDeviceTestBase { mInner.newVmConfigBuilder("assets/vm_config_io.json") .debugLevel(DebugLevel.FULL) .build(); - List transferRates = new ArrayList<>(); + List transferRates = new ArrayList<>(IO_TEST_TRIAL_COUNT); for (int i = 0; i < IO_TEST_TRIAL_COUNT; ++i) { int port = (mProtectedVm ? 5666 : 6666) + i; @@ -196,7 +200,7 @@ public class MicrodroidBenchmarks extends MicrodroidDeviceTestBase { VirtualMachine vm = mInner.getVirtualMachineManager().get(vmName); BenchmarkVmListener.create(new VsockListener(transferRates, port)).runToFinish(TAG, vm); } - reportMetrics(transferRates, "vsock/transfer_host_to_vm_", "_mb_per_sec"); + reportMetrics(transferRates, "vsock/transfer_host_to_vm", "mb_per_sec"); } @Test @@ -214,7 +218,7 @@ public class MicrodroidBenchmarks extends MicrodroidDeviceTestBase { mInner.newVmConfigBuilder("assets/vm_config_io.json") .debugLevel(DebugLevel.FULL) .build(); - List readRates = new ArrayList<>(); + List readRates = new ArrayList<>(IO_TEST_TRIAL_COUNT); for (int i = 0; i < IO_TEST_TRIAL_COUNT + 1; ++i) { if (i == 1) { @@ -230,33 +234,15 @@ public class MicrodroidBenchmarks extends MicrodroidDeviceTestBase { .runToFinish(TAG, vm); } reportMetrics( - readRates, - isRand ? "virtio-blk/rand_read_" : "virtio-blk/seq_read_", - "_mb_per_sec"); + readRates, isRand ? "virtio-blk/rand_read" : "virtio-blk/seq_read", "mb_per_sec"); } - private void reportMetrics(List data, String base, String suffix) { - double sum = 0; - double min = Double.MAX_VALUE; - double max = Double.MIN_VALUE; - for (double d : data) { - sum += d; - if (min > d) min = d; - if (max < d) max = d; - } - double avg = sum / data.size(); - double sqSum = 0; - for (double d : data) { - sqSum += (d - avg) * (d - avg); - } - double stdDev = Math.sqrt(sqSum / (data.size() - 1)); - + private void reportMetrics(List metrics, String name, String unit) { + Map stats = mMetricsProcessor.computeStats(metrics, name, unit); Bundle bundle = new Bundle(); - String prefix = METRIC_NAME_PREFIX + base; - bundle.putDouble(prefix + "min" + suffix, min); - bundle.putDouble(prefix + "max" + suffix, max); - bundle.putDouble(prefix + "average" + suffix, avg); - bundle.putDouble(prefix + "stdev" + suffix, stdDev); + for (Map.Entry entry : stats.entrySet()) { + bundle.putDouble(entry.getKey(), entry.getValue()); + } mInstrumentation.sendStatus(0, bundle); } diff --git a/tests/benchmark_hostside/Android.bp b/tests/benchmark_hostside/Android.bp index 354a2603..c5c75710 100644 --- a/tests/benchmark_hostside/Android.bp +++ b/tests/benchmark_hostside/Android.bp @@ -12,6 +12,7 @@ java_test_host { ], static_libs: [ "MicrodroidHostTestHelper", + "MicrodroidTestHelper", ], test_suites: [ "general-tests", diff --git a/tests/benchmark_hostside/java/android/avf/test/AVFHostTestCase.java b/tests/benchmark_hostside/java/android/avf/test/AVFHostTestCase.java index 8e9da55a..143a35a8 100644 --- a/tests/benchmark_hostside/java/android/avf/test/AVFHostTestCase.java +++ b/tests/benchmark_hostside/java/android/avf/test/AVFHostTestCase.java @@ -16,8 +16,6 @@ package android.avf.test; -import android.platform.test.annotations.RootPermissionTest; - import static com.android.tradefed.testtype.DeviceJUnit4ClassRunner.TestMetrics; import static com.google.common.truth.Truth.assertWithMessage; @@ -25,8 +23,11 @@ import static com.google.common.truth.TruthJUnit.assume; import static org.junit.Assume.assumeTrue; +import android.platform.test.annotations.RootPermissionTest; + import com.android.microdroid.test.CommandRunner; import com.android.microdroid.test.MicrodroidHostTestCaseBase; +import com.android.microdroid.test.common.MetricsProcessor; import com.android.tradefed.log.LogUtil.CLog; import com.android.tradefed.testtype.DeviceJUnit4ClassRunner; import com.android.tradefed.util.CommandResult; @@ -36,6 +37,9 @@ import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -61,6 +65,8 @@ public final class AVFHostTestCase extends MicrodroidHostTestCaseBase { private static final int ROUND_COUNT = 5; private static final String METRIC_PREFIX = "avf_perf/hostside/"; + private final MetricsProcessor mMetricsProcessor = new MetricsProcessor(METRIC_PREFIX); + @Before public void setUp() throws Exception { testIfDeviceIsCapable(getDevice()); @@ -80,11 +86,10 @@ public final class AVFHostTestCase extends MicrodroidHostTestCaseBase { @Test public void testBootEnableAndDisablePKVM() throws Exception { - testPKVMStatusSwitchSupported(); - double[] bootWithPKVMEnableTime = new double[ROUND_COUNT]; - double[] bootWithoutPKVMEnableTime = new double[ROUND_COUNT]; + List bootWithPKVMEnableTime = new ArrayList<>(ROUND_COUNT); + List bootWithoutPKVMEnableTime = new ArrayList<>(ROUND_COUNT); for (int round = 0; round < ROUND_COUNT; ++round) { @@ -93,7 +98,7 @@ public final class AVFHostTestCase extends MicrodroidHostTestCaseBase { rebootFromBootloaderAndWaitBootCompleted(); long elapsedWithPKVMEnable = System.nanoTime() - start; double elapsedSec = elapsedWithPKVMEnable / NANOS_IN_SEC; - bootWithPKVMEnableTime[round] = elapsedSec; + bootWithPKVMEnableTime.add(elapsedSec); CLog.i("Boot time with PKVM enable took " + elapsedSec + "s"); setPKVMStatusWithRebootToBootloader(false); @@ -101,20 +106,20 @@ public final class AVFHostTestCase extends MicrodroidHostTestCaseBase { rebootFromBootloaderAndWaitBootCompleted(); long elapsedWithoutPKVMEnable = System.nanoTime() - start; elapsedSec = elapsedWithoutPKVMEnable / NANOS_IN_SEC; - bootWithoutPKVMEnableTime[round] = elapsedSec; + bootWithoutPKVMEnableTime.add(elapsedSec); CLog.i("Boot time with PKVM disable took " + elapsedSec + "s"); } - reportMetric("boot_time_with_pkvm_enable", "s", bootWithPKVMEnableTime); - reportMetric("boot_time_with_pkvm_disable", "s", bootWithoutPKVMEnableTime); + reportMetric(bootWithPKVMEnableTime, "boot_time_with_pkvm_enable", "s"); + reportMetric(bootWithoutPKVMEnableTime, "boot_time_with_pkvm_disable", "s"); } @Test public void testBootWithAndWithoutCompOS() throws Exception { assume().withMessage("Skip on CF; too slow").that(isCuttlefish()).isFalse(); - double[] bootWithCompOsTime = new double[ROUND_COUNT]; - double[] bootWithoutCompOsTime = new double[ROUND_COUNT]; + List bootWithCompOsTime = new ArrayList<>(ROUND_COUNT); + List bootWithoutCompOsTime = new ArrayList<>(ROUND_COUNT); for (int round = 0; round < ROUND_COUNT; ++round) { @@ -125,7 +130,7 @@ public final class AVFHostTestCase extends MicrodroidHostTestCaseBase { rebootAndWaitBootCompleted(); long elapsedWithCompOS = System.nanoTime() - start; double elapsedSec = elapsedWithCompOS / NANOS_IN_SEC; - bootWithCompOsTime[round] = elapsedSec; + bootWithCompOsTime.add(elapsedSec); CLog.i("Boot time with compilation OS took " + elapsedSec + "s"); // Boot time without compilation OS test. @@ -134,12 +139,12 @@ public final class AVFHostTestCase extends MicrodroidHostTestCaseBase { rebootAndWaitBootCompleted(); long elapsedWithoutCompOS = System.nanoTime() - start; elapsedSec = elapsedWithoutCompOS / NANOS_IN_SEC; - bootWithoutCompOsTime[round] = elapsedSec; + bootWithoutCompOsTime.add(elapsedSec); CLog.i("Boot time without compilation OS took " + elapsedSec + "s"); } - reportMetric("boot_time_with_compos", "s", bootWithCompOsTime); - reportMetric("boot_time_without_compos", "s", bootWithoutCompOsTime); + reportMetric(bootWithCompOsTime, "boot_time_with_compos", "s"); + reportMetric(bootWithoutCompOsTime, "boot_time_without_compos", "s"); } private void testPKVMStatusSwitchSupported() throws Exception { @@ -154,31 +159,12 @@ public final class AVFHostTestCase extends MicrodroidHostTestCaseBase { assumeTrue(!result.getStderr().contains("Invalid oem command")); } - private void reportMetric(String name, String unit, double[] values) { - double sum = 0; - double min = Double.MAX_VALUE; - double max = Double.MIN_VALUE; - - for (double val : values) { - sum += val; - min = val < min ? val : min; - max = val > max ? val : max; - } - - double average = sum / values.length; - - double variance = 0; - for (double val : values) { - final double tmp = val - average; - variance += tmp * tmp; - } - double stdev = Math.sqrt(variance / (double) (values.length - 1)); - + private void reportMetric(List data, String name, String unit) { + Map stats = mMetricsProcessor.computeStats(data, name, unit); TestMetrics metrics = new TestMetrics(); - metrics.addTestMetric(METRIC_PREFIX + name + "_average_" + unit, Double.toString(average)); - metrics.addTestMetric(METRIC_PREFIX + name + "_min_" + unit, Double.toString(min)); - metrics.addTestMetric(METRIC_PREFIX + name + "_max_" + unit, Double.toString(max)); - metrics.addTestMetric(METRIC_PREFIX + name + "_stdev_" + unit, Double.toString(stdev)); + for (Map.Entry entry : stats.entrySet()) { + metrics.addTestMetric(entry.getKey(), Double.toString(entry.getValue())); + } } private void setPKVMStatusWithRebootToBootloader(boolean isEnable) throws Exception { diff --git a/tests/helper/Android.bp b/tests/helper/Android.bp index f8dcef72..6d769e62 100644 --- a/tests/helper/Android.bp +++ b/tests/helper/Android.bp @@ -8,9 +8,15 @@ java_library_static { host_supported: true, } +java_library_static { + name: "MicrodroidTestHelper", + srcs: ["src/java/com/android/microdroid/test/common/*.java"], + host_supported: true, +} + java_library_static { name: "MicrodroidDeviceTestHelper", - srcs: ["src/java/com/android/microdroid/**/*.java"], + srcs: ["src/java/com/android/microdroid/test/*.java"], static_libs: [ "androidx.test.runner", "androidx.test.ext.junit", diff --git a/tests/helper/src/java/com/android/microdroid/test/common/MetricsProcessor.java b/tests/helper/src/java/com/android/microdroid/test/common/MetricsProcessor.java new file mode 100644 index 00000000..c12b07dd --- /dev/null +++ b/tests/helper/src/java/com/android/microdroid/test/common/MetricsProcessor.java @@ -0,0 +1,60 @@ +/* + * Copyright (C) 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.microdroid.test.common; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** This class processes the metrics for both device tests and host tests. */ +public final class MetricsProcessor { + private final String mPrefix; + + public MetricsProcessor(String prefix) { + mPrefix = prefix; + } + + /** + * Computes the min, max, average and standard deviation of the given metrics and saves them in + * a {@link Map} with the corresponding keys equal to [mPrefix + name + + * _[min|max|average|stdev]_ + unit]. + */ + public Map computeStats(List metrics, String name, String unit) { + double sum = 0; + double min = Double.MAX_VALUE; + double max = Double.MIN_VALUE; + for (double d : metrics) { + sum += d; + if (min > d) min = d; + if (max < d) max = d; + } + double avg = sum / metrics.size(); + double sqSum = 0; + for (double d : metrics) { + sqSum += (d - avg) * (d - avg); + } + double stdDev = Math.sqrt(sqSum / (metrics.size() - 1)); + + Map stats = new HashMap(); + String prefix = mPrefix + name; + stats.put(prefix + "_min_" + unit, min); + stats.put(prefix + "_max_" + unit, max); + stats.put(prefix + "_average_" + unit, avg); + stats.put(prefix + "_stdev_" + unit, stdDev); + return stats; + } +}