From 678a10600c6146bde78baa925e86dc99131c5d8a Mon Sep 17 00:00:00 2001 From: Michael Butler Date: Thu, 19 Mar 2020 17:10:34 -0700 Subject: [PATCH] Remove extra tests from NNAPI VTS validation tests Do not run validation on "inputs_as_internal" and "all_tensors_as_inputs" variants. Bug: 138149072 Bug: 149840439 Test: mma Test: VtsHalNeuralnetworksV1_*TargetTest Change-Id: I0699ed6703e48b6c4bc0e7a392b79c12770f04c9 --- neuralnetworks/1.0/vts/functional/GeneratedTestHarness.cpp | 4 ++++ neuralnetworks/1.0/vts/functional/GeneratedTestHarness.h | 3 +++ neuralnetworks/1.0/vts/functional/VtsHalNeuralnetworks.cpp | 7 ++++++- neuralnetworks/1.1/vts/functional/GeneratedTestHarness.cpp | 4 ++++ neuralnetworks/1.1/vts/functional/GeneratedTestHarness.h | 3 +++ neuralnetworks/1.1/vts/functional/VtsHalNeuralnetworks.cpp | 7 ++++++- neuralnetworks/1.2/vts/functional/GeneratedTestHarness.cpp | 4 ++++ neuralnetworks/1.2/vts/functional/GeneratedTestHarness.h | 3 +++ neuralnetworks/1.2/vts/functional/VtsHalNeuralnetworks.cpp | 7 ++++++- neuralnetworks/1.3/vts/functional/GeneratedTestHarness.cpp | 4 ++++ neuralnetworks/1.3/vts/functional/GeneratedTestHarness.h | 3 +++ neuralnetworks/1.3/vts/functional/VtsHalNeuralnetworks.cpp | 7 ++++++- 12 files changed, 52 insertions(+), 4 deletions(-) diff --git a/neuralnetworks/1.0/vts/functional/GeneratedTestHarness.cpp b/neuralnetworks/1.0/vts/functional/GeneratedTestHarness.cpp index e28605dca2..87de9c605f 100644 --- a/neuralnetworks/1.0/vts/functional/GeneratedTestHarness.cpp +++ b/neuralnetworks/1.0/vts/functional/GeneratedTestHarness.cpp @@ -158,6 +158,10 @@ std::vector getNamedModels(const FilterFn& filter) { return TestModelManager::get().getTestModels(filter); } +std::vector getNamedModels(const FilterNameFn& filter) { + return TestModelManager::get().getTestModels(filter); +} + std::string printGeneratedTest(const testing::TestParamInfo& info) { const auto& [namedDevice, namedModel] = info.param; return gtestCompliantName(getName(namedDevice) + "_" + getName(namedModel)); diff --git a/neuralnetworks/1.0/vts/functional/GeneratedTestHarness.h b/neuralnetworks/1.0/vts/functional/GeneratedTestHarness.h index f230a028f3..1a55c2f9c8 100644 --- a/neuralnetworks/1.0/vts/functional/GeneratedTestHarness.h +++ b/neuralnetworks/1.0/vts/functional/GeneratedTestHarness.h @@ -37,6 +37,9 @@ class GeneratedTestBase : public testing::TestWithParam { using FilterFn = std::function; std::vector getNamedModels(const FilterFn& filter); +using FilterNameFn = std::function; +std::vector getNamedModels(const FilterNameFn& filter); + std::string printGeneratedTest(const testing::TestParamInfo& info); #define INSTANTIATE_GENERATED_TEST(TestSuite, filter) \ diff --git a/neuralnetworks/1.0/vts/functional/VtsHalNeuralnetworks.cpp b/neuralnetworks/1.0/vts/functional/VtsHalNeuralnetworks.cpp index cb2225025b..70170b39ac 100644 --- a/neuralnetworks/1.0/vts/functional/VtsHalNeuralnetworks.cpp +++ b/neuralnetworks/1.0/vts/functional/VtsHalNeuralnetworks.cpp @@ -134,6 +134,11 @@ TEST_P(ValidationTest, Test) { validateEverything(kDevice, model, request); } -INSTANTIATE_GENERATED_TEST(ValidationTest, [](const test_helper::TestModel&) { return true; }); +INSTANTIATE_GENERATED_TEST(ValidationTest, [](const std::string& testName) { + // Skip validation for the "inputs_as_internal" and "all_tensors_as_inputs" + // generated tests. + return testName.find("inputs_as_internal") == std::string::npos && + testName.find("all_tensors_as_inputs") == std::string::npos; +}); } // namespace android::hardware::neuralnetworks::V1_0::vts::functional diff --git a/neuralnetworks/1.1/vts/functional/GeneratedTestHarness.cpp b/neuralnetworks/1.1/vts/functional/GeneratedTestHarness.cpp index cee15a35a1..7353c61006 100644 --- a/neuralnetworks/1.1/vts/functional/GeneratedTestHarness.cpp +++ b/neuralnetworks/1.1/vts/functional/GeneratedTestHarness.cpp @@ -166,6 +166,10 @@ std::vector getNamedModels(const FilterFn& filter) { return TestModelManager::get().getTestModels(filter); } +std::vector getNamedModels(const FilterNameFn& filter) { + return TestModelManager::get().getTestModels(filter); +} + std::string printGeneratedTest(const testing::TestParamInfo& info) { const auto& [namedDevice, namedModel] = info.param; return gtestCompliantName(getName(namedDevice) + "_" + getName(namedModel)); diff --git a/neuralnetworks/1.1/vts/functional/GeneratedTestHarness.h b/neuralnetworks/1.1/vts/functional/GeneratedTestHarness.h index cf449ea42d..4b1a96e00d 100644 --- a/neuralnetworks/1.1/vts/functional/GeneratedTestHarness.h +++ b/neuralnetworks/1.1/vts/functional/GeneratedTestHarness.h @@ -37,6 +37,9 @@ class GeneratedTestBase : public testing::TestWithParam { using FilterFn = std::function; std::vector getNamedModels(const FilterFn& filter); +using FilterNameFn = std::function; +std::vector getNamedModels(const FilterNameFn& filter); + std::string printGeneratedTest(const testing::TestParamInfo& info); #define INSTANTIATE_GENERATED_TEST(TestSuite, filter) \ diff --git a/neuralnetworks/1.1/vts/functional/VtsHalNeuralnetworks.cpp b/neuralnetworks/1.1/vts/functional/VtsHalNeuralnetworks.cpp index d56d40b2ba..15d2260ec0 100644 --- a/neuralnetworks/1.1/vts/functional/VtsHalNeuralnetworks.cpp +++ b/neuralnetworks/1.1/vts/functional/VtsHalNeuralnetworks.cpp @@ -137,6 +137,11 @@ TEST_P(ValidationTest, Test) { validateEverything(kDevice, model, request); } -INSTANTIATE_GENERATED_TEST(ValidationTest, [](const test_helper::TestModel&) { return true; }); +INSTANTIATE_GENERATED_TEST(ValidationTest, [](const std::string& testName) { + // Skip validation for the "inputs_as_internal" and "all_tensors_as_inputs" + // generated tests. + return testName.find("inputs_as_internal") == std::string::npos && + testName.find("all_tensors_as_inputs") == std::string::npos; +}); } // namespace android::hardware::neuralnetworks::V1_1::vts::functional diff --git a/neuralnetworks/1.2/vts/functional/GeneratedTestHarness.cpp b/neuralnetworks/1.2/vts/functional/GeneratedTestHarness.cpp index 3ab01351e9..573545a049 100644 --- a/neuralnetworks/1.2/vts/functional/GeneratedTestHarness.cpp +++ b/neuralnetworks/1.2/vts/functional/GeneratedTestHarness.cpp @@ -382,6 +382,10 @@ std::vector getNamedModels(const FilterFn& filter) { return TestModelManager::get().getTestModels(filter); } +std::vector getNamedModels(const FilterNameFn& filter) { + return TestModelManager::get().getTestModels(filter); +} + std::string printGeneratedTest(const testing::TestParamInfo& info) { const auto& [namedDevice, namedModel] = info.param; return gtestCompliantName(getName(namedDevice) + "_" + getName(namedModel)); diff --git a/neuralnetworks/1.2/vts/functional/GeneratedTestHarness.h b/neuralnetworks/1.2/vts/functional/GeneratedTestHarness.h index dfc980c169..98295ff64a 100644 --- a/neuralnetworks/1.2/vts/functional/GeneratedTestHarness.h +++ b/neuralnetworks/1.2/vts/functional/GeneratedTestHarness.h @@ -41,6 +41,9 @@ class GeneratedTestBase : public testing::TestWithParam { using FilterFn = std::function; std::vector getNamedModels(const FilterFn& filter); +using FilterNameFn = std::function; +std::vector getNamedModels(const FilterNameFn& filter); + std::string printGeneratedTest(const testing::TestParamInfo& info); #define INSTANTIATE_GENERATED_TEST(TestSuite, filter) \ diff --git a/neuralnetworks/1.2/vts/functional/VtsHalNeuralnetworks.cpp b/neuralnetworks/1.2/vts/functional/VtsHalNeuralnetworks.cpp index 4fbd0e270f..69bce29011 100644 --- a/neuralnetworks/1.2/vts/functional/VtsHalNeuralnetworks.cpp +++ b/neuralnetworks/1.2/vts/functional/VtsHalNeuralnetworks.cpp @@ -161,7 +161,12 @@ TEST_P(ValidationTest, Test) { } } -INSTANTIATE_GENERATED_TEST(ValidationTest, [](const test_helper::TestModel&) { return true; }); +INSTANTIATE_GENERATED_TEST(ValidationTest, [](const std::string& testName) { + // Skip validation for the "inputs_as_internal" and "all_tensors_as_inputs" + // generated tests. + return testName.find("inputs_as_internal") == std::string::npos && + testName.find("all_tensors_as_inputs") == std::string::npos; +}); sp getPreparedModel_1_2(const sp& callback) { sp preparedModelV1_0 = callback->getPreparedModel(); diff --git a/neuralnetworks/1.3/vts/functional/GeneratedTestHarness.cpp b/neuralnetworks/1.3/vts/functional/GeneratedTestHarness.cpp index aae58bfb3e..9eecbc78d1 100644 --- a/neuralnetworks/1.3/vts/functional/GeneratedTestHarness.cpp +++ b/neuralnetworks/1.3/vts/functional/GeneratedTestHarness.cpp @@ -891,6 +891,10 @@ std::vector getNamedModels(const FilterFn& filter) { return TestModelManager::get().getTestModels(filter); } +std::vector getNamedModels(const FilterNameFn& filter) { + return TestModelManager::get().getTestModels(filter); +} + std::string printGeneratedTest(const testing::TestParamInfo& info) { const auto& [namedDevice, namedModel] = info.param; return gtestCompliantName(getName(namedDevice) + "_" + getName(namedModel)); diff --git a/neuralnetworks/1.3/vts/functional/GeneratedTestHarness.h b/neuralnetworks/1.3/vts/functional/GeneratedTestHarness.h index 834d335f50..065f7ef2b8 100644 --- a/neuralnetworks/1.3/vts/functional/GeneratedTestHarness.h +++ b/neuralnetworks/1.3/vts/functional/GeneratedTestHarness.h @@ -41,6 +41,9 @@ class GeneratedTestBase : public testing::TestWithParam { using FilterFn = std::function; std::vector getNamedModels(const FilterFn& filter); +using FilterNameFn = std::function; +std::vector getNamedModels(const FilterNameFn& filter); + std::string printGeneratedTest(const testing::TestParamInfo& info); #define INSTANTIATE_GENERATED_TEST(TestSuite, filter) \ diff --git a/neuralnetworks/1.3/vts/functional/VtsHalNeuralnetworks.cpp b/neuralnetworks/1.3/vts/functional/VtsHalNeuralnetworks.cpp index 5b07034296..eda867e17a 100644 --- a/neuralnetworks/1.3/vts/functional/VtsHalNeuralnetworks.cpp +++ b/neuralnetworks/1.3/vts/functional/VtsHalNeuralnetworks.cpp @@ -185,7 +185,12 @@ TEST_P(ValidationTest, Test) { } } -INSTANTIATE_GENERATED_TEST(ValidationTest, [](const test_helper::TestModel&) { return true; }); +INSTANTIATE_GENERATED_TEST(ValidationTest, [](const std::string& testName) { + // Skip validation for the "inputs_as_internal" and "all_tensors_as_inputs" + // generated tests. + return testName.find("inputs_as_internal") == std::string::npos && + testName.find("all_tensors_as_inputs") == std::string::npos; +}); sp getPreparedModel_1_3(const sp& callback) { sp preparedModelV1_0 = callback->getPreparedModel();