From 2d74d7dfae62a02dccd4ed313f2348a505f7afaf Mon Sep 17 00:00:00 2001 From: Miao Wang Date: Tue, 25 Feb 2020 15:54:08 -0800 Subject: [PATCH 1/3] Add more tests exercising IPreparedModel::executeFenced API - executeFenced with device memory - executeFenced with depending sync fences Bug: 148979873 Test: mm Test: VtsHalNeuralnetworksV1_3TargetTest Change-Id: I772c5c85f75cee56e2af06470c2de5b810f8078d Merged-In: I772c5c85f75cee56e2af06470c2de5b810f8078d (cherry picked from commit 34dfa2f74cf79ce66afd08b681bb2bb3a136c5d9) --- .../vts/functional/GeneratedTestHarness.cpp | 27 ++++++++++++------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/neuralnetworks/1.3/vts/functional/GeneratedTestHarness.cpp b/neuralnetworks/1.3/vts/functional/GeneratedTestHarness.cpp index 8c9393b030..3e2be8a346 100644 --- a/neuralnetworks/1.3/vts/functional/GeneratedTestHarness.cpp +++ b/neuralnetworks/1.3/vts/functional/GeneratedTestHarness.cpp @@ -626,21 +626,28 @@ void EvaluatePreparedModel(const sp& device, const sp& ErrorStatus result; hidl_handle syncFenceHandle; sp fencedCallback; - Return ret = preparedModel->executeFenced( - request, {}, testConfig.measureTiming, {}, loopTimeoutDuration, {}, - [&result, &syncFenceHandle, &fencedCallback]( - ErrorStatus error, const hidl_handle& handle, - const sp& callback) { - result = error; - syncFenceHandle = handle; - fencedCallback = callback; - }); + auto callbackFunc = [&result, &syncFenceHandle, &fencedCallback]( + ErrorStatus error, const hidl_handle& handle, + const sp& callback) { + result = error; + syncFenceHandle = handle; + fencedCallback = callback; + }; + Return ret = + preparedModel->executeFenced(request, {}, testConfig.measureTiming, {}, + loopTimeoutDuration, {}, callbackFunc); ASSERT_TRUE(ret.isOk()); if (result != ErrorStatus::NONE) { ASSERT_EQ(syncFenceHandle.getNativeHandle(), nullptr); ASSERT_EQ(fencedCallback, nullptr); executionStatus = ErrorStatus::GENERAL_FAILURE; } else if (syncFenceHandle.getNativeHandle()) { + // If a sync fence is returned, try start another run waiting for the sync fence. + ret = preparedModel->executeFenced(request, {syncFenceHandle}, + testConfig.measureTiming, {}, + loopTimeoutDuration, {}, callbackFunc); + ASSERT_TRUE(ret.isOk()); + ASSERT_EQ(result, ErrorStatus::NONE); waitForSyncFence(syncFenceHandle.getNativeHandle()->data[0]); } if (result == ErrorStatus::NONE) { @@ -744,7 +751,7 @@ void EvaluatePreparedModel(const sp& device, const sp& case TestKind::MEMORY_DOMAIN: { outputTypesList = {OutputType::FULLY_SPECIFIED}; measureTimingList = {MeasureTiming::NO}; - executorList = {Executor::ASYNC, Executor::SYNC}; + executorList = {Executor::ASYNC, Executor::SYNC, Executor::FENCED}; memoryType = MemoryType::DEVICE; } break; case TestKind::FENCED_COMPUTE: { From 406890769a66fea031ca1731072fe7137ce02a24 Mon Sep 17 00:00:00 2001 From: Miao Wang Date: Tue, 25 Feb 2020 16:57:53 -0800 Subject: [PATCH 2/3] Add tests to make sure executeFenced validate unspecified output shapes Bug: 148979873 Test: mm Test: VtsHalNeuralnetworksV1_3TargetTest Change-Id: Ib4960c71ca46ca034777cc7b02d7d2885a98691d Merged-In: Ib4960c71ca46ca034777cc7b02d7d2885a98691d (cherry picked from commit fa161c7a0a2d1719680248ccc6d8978209a13d19) --- .../vts/functional/GeneratedTestHarness.cpp | 27 ++++++++++++++----- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/neuralnetworks/1.3/vts/functional/GeneratedTestHarness.cpp b/neuralnetworks/1.3/vts/functional/GeneratedTestHarness.cpp index 3e2be8a346..83a8d94ba5 100644 --- a/neuralnetworks/1.3/vts/functional/GeneratedTestHarness.cpp +++ b/neuralnetworks/1.3/vts/functional/GeneratedTestHarness.cpp @@ -640,7 +640,7 @@ void EvaluatePreparedModel(const sp& device, const sp& if (result != ErrorStatus::NONE) { ASSERT_EQ(syncFenceHandle.getNativeHandle(), nullptr); ASSERT_EQ(fencedCallback, nullptr); - executionStatus = ErrorStatus::GENERAL_FAILURE; + executionStatus = result; } else if (syncFenceHandle.getNativeHandle()) { // If a sync fence is returned, try start another run waiting for the sync fence. ret = preparedModel->executeFenced(request, {syncFenceHandle}, @@ -663,9 +663,7 @@ void EvaluatePreparedModel(const sp& device, const sp& } } - // The driver is allowed to reject executeFenced, and if they do, we should skip. - if ((testConfig.outputType != OutputType::FULLY_SPECIFIED || - testConfig.executor == Executor::FENCED) && + if (testConfig.outputType != OutputType::FULLY_SPECIFIED && executionStatus == ErrorStatus::GENERAL_FAILURE) { if (skipped != nullptr) { *skipped = true; @@ -698,12 +696,22 @@ void EvaluatePreparedModel(const sp& device, const sp& outputShapes.size() == testModel.main.outputIndexes.size()); break; case OutputType::UNSPECIFIED: + if (testConfig.executor == Executor::FENCED) { + // For Executor::FENCED, the output shape must be fully specified. + ASSERT_EQ(ErrorStatus::INVALID_ARGUMENT, executionStatus); + return; + } // If the model output operands are not fully specified, outputShapes must have // the same number of elements as the number of outputs. ASSERT_EQ(ErrorStatus::NONE, executionStatus); ASSERT_EQ(outputShapes.size(), testModel.main.outputIndexes.size()); break; case OutputType::INSUFFICIENT: + if (testConfig.executor == Executor::FENCED) { + // For Executor::FENCED, the output shape must be fully specified. + ASSERT_EQ(ErrorStatus::INVALID_ARGUMENT, executionStatus); + return; + } ASSERT_EQ(ErrorStatus::OUTPUT_INSUFFICIENT_SIZE, executionStatus); ASSERT_EQ(outputShapes.size(), testModel.main.outputIndexes.size()); ASSERT_FALSE(outputShapes[0].isSufficient); @@ -746,7 +754,7 @@ void EvaluatePreparedModel(const sp& device, const sp& case TestKind::DYNAMIC_SHAPE: { outputTypesList = {OutputType::UNSPECIFIED, OutputType::INSUFFICIENT}; measureTimingList = {MeasureTiming::NO, MeasureTiming::YES}; - executorList = {Executor::ASYNC, Executor::SYNC, Executor::BURST}; + executorList = {Executor::ASYNC, Executor::SYNC, Executor::BURST, Executor::FENCED}; } break; case TestKind::MEMORY_DOMAIN: { outputTypesList = {OutputType::FULLY_SPECIFIED}; @@ -928,8 +936,13 @@ INSTANTIATE_GENERATED_TEST(DynamicOutputShapeTest, [](const TestModel& testModel INSTANTIATE_GENERATED_TEST(MemoryDomainTest, [](const TestModel& testModel) { return !testModel.expectFailure; }); -INSTANTIATE_GENERATED_TEST(FencedComputeTest, - [](const TestModel& testModel) { return !testModel.expectFailure; }); +INSTANTIATE_GENERATED_TEST(FencedComputeTest, [](const TestModel& testModel) { + return !testModel.expectFailure && + std::all_of(testModel.main.outputIndexes.begin(), testModel.main.outputIndexes.end(), + [&testModel](uint32_t index) { + return testModel.main.operands[index].data.size() > 0; + }); +}); INSTANTIATE_GENERATED_TEST(QuantizationCouplingTest, [](const TestModel& testModel) { return testModel.hasQuant8CoupledOperands() && testModel.main.operations.size() == 1; From d09c0eef0b1c44c471884b74ab1c23d4919ba7e0 Mon Sep 17 00:00:00 2001 From: Michael Butler Date: Thu, 12 Mar 2020 15:12:23 -0700 Subject: [PATCH 3/3] NNAPI VTS: decouple 1.2 tests from 1.3 types Bug: 141718368 Test: mma Test: VtsHalNeuralnetworksV1_*TargetTest Change-Id: Ied7b95e146e4d2d95622642bfbb959cf134b5a7a Merged-In: Ied7b95e146e4d2d95622642bfbb959cf134b5a7a (cherry picked from commit 4876af1ba1983bd1b102ba35928051820279faa3) --- neuralnetworks/1.2/vts/functional/Android.bp | 14 ++++++++------ .../1.2/vts/functional/CompilationCachingTests.cpp | 1 - .../1.2/vts/functional/GeneratedTestHarness.cpp | 3 +-- .../1.2/vts/functional/ValidateBurst.cpp | 6 ++---- .../1.2/vts/functional/ValidateRequest.cpp | 3 +-- .../1.3/vts/functional/ValidateBurst.cpp | 7 ++----- 6 files changed, 14 insertions(+), 20 deletions(-) diff --git a/neuralnetworks/1.2/vts/functional/Android.bp b/neuralnetworks/1.2/vts/functional/Android.bp index 31a1a819ee..7c1faeef68 100644 --- a/neuralnetworks/1.2/vts/functional/Android.bp +++ b/neuralnetworks/1.2/vts/functional/Android.bp @@ -28,7 +28,7 @@ cc_library_static { ], header_libs: [ "libbase_headers", - ] + ], } cc_test { @@ -39,9 +39,9 @@ cc_test { "CompilationCachingTests.cpp", "GeneratedTestHarness.cpp", "TestAssertions.cpp", + "ValidateBurst.cpp", "ValidateModel.cpp", "ValidateRequest.cpp", - "ValidateBurst.cpp", "VtsHalNeuralnetworks.cpp", ], local_include_dirs: ["include"], @@ -50,18 +50,17 @@ cc_test { "libnativewindow", ], static_libs: [ + "VtsHalNeuralNetworksV1_0_utils", + "VtsHalNeuralNetworksV1_2Callbacks", "android.hardware.neuralnetworks@1.0", "android.hardware.neuralnetworks@1.1", "android.hardware.neuralnetworks@1.2", - "android.hardware.neuralnetworks@1.3", "android.hidl.allocator@1.0", "android.hidl.memory@1.0", "libgmock", "libhidlmemory", "libneuralnetworks_generated_test_harness", "libneuralnetworks_utils", - "VtsHalNeuralNetworksV1_0_utils", - "VtsHalNeuralNetworksV1_2Callbacks", ], whole_static_libs: [ "neuralnetworks_generated_V1_0_example", @@ -71,5 +70,8 @@ cc_test { header_libs: [ "libneuralnetworks_headers", ], - test_suites: ["general-tests", "vts-core"], + test_suites: [ + "general-tests", + "vts-core", + ], } diff --git a/neuralnetworks/1.2/vts/functional/CompilationCachingTests.cpp b/neuralnetworks/1.2/vts/functional/CompilationCachingTests.cpp index 10dec791cf..449b8f369d 100644 --- a/neuralnetworks/1.2/vts/functional/CompilationCachingTests.cpp +++ b/neuralnetworks/1.2/vts/functional/CompilationCachingTests.cpp @@ -32,7 +32,6 @@ #include "GeneratedTestHarness.h" #include "MemoryUtils.h" #include "TestHarness.h" -#include "Utils.h" #include "VtsHalNeuralnetworks.h" // Forward declaration of the mobilenet generated test models in diff --git a/neuralnetworks/1.2/vts/functional/GeneratedTestHarness.cpp b/neuralnetworks/1.2/vts/functional/GeneratedTestHarness.cpp index 4c8fede8b2..3ab01351e9 100644 --- a/neuralnetworks/1.2/vts/functional/GeneratedTestHarness.cpp +++ b/neuralnetworks/1.2/vts/functional/GeneratedTestHarness.cpp @@ -43,7 +43,6 @@ #include "ExecutionBurstController.h" #include "MemoryUtils.h" #include "TestHarness.h" -#include "Utils.h" #include "VtsHalNeuralnetworks.h" namespace android::hardware::neuralnetworks::V1_2::vts::functional { @@ -273,7 +272,7 @@ void EvaluatePreparedModel(const sp& preparedModel, const TestMo int n; std::tie(n, outputShapes, timing, std::ignore) = controller->compute(request, testConfig.measureTiming, keys); - executionStatus = nn::convertToV1_0(nn::convertResultCodeToErrorStatus(n)); + executionStatus = nn::legacyConvertResultCodeToErrorStatus(n); break; } diff --git a/neuralnetworks/1.2/vts/functional/ValidateBurst.cpp b/neuralnetworks/1.2/vts/functional/ValidateBurst.cpp index ec9629bccb..cc9d8048d0 100644 --- a/neuralnetworks/1.2/vts/functional/ValidateBurst.cpp +++ b/neuralnetworks/1.2/vts/functional/ValidateBurst.cpp @@ -23,7 +23,6 @@ #include "ExecutionBurstServer.h" #include "GeneratedTestHarness.h" #include "TestHarness.h" -#include "Utils.h" #include #include @@ -296,8 +295,7 @@ static void validateBurstFmqLength(const sp& preparedModel, // collect serialized result by running regular burst const auto [nRegular, outputShapesRegular, timingRegular, fallbackRegular] = controllerRegular->compute(request, MeasureTiming::NO, keys); - const ErrorStatus statusRegular = - nn::convertToV1_0(nn::convertResultCodeToErrorStatus(nRegular)); + const ErrorStatus statusRegular = nn::legacyConvertResultCodeToErrorStatus(nRegular); EXPECT_FALSE(fallbackRegular); // skip test if regular burst output isn't useful for testing a failure @@ -313,7 +311,7 @@ static void validateBurstFmqLength(const sp& preparedModel, // large enough to return the serialized result const auto [nSmall, outputShapesSmall, timingSmall, fallbackSmall] = controllerSmall->compute(request, MeasureTiming::NO, keys); - const ErrorStatus statusSmall = nn::convertToV1_0(nn::convertResultCodeToErrorStatus(nSmall)); + const ErrorStatus statusSmall = nn::legacyConvertResultCodeToErrorStatus(nSmall); EXPECT_NE(ErrorStatus::NONE, statusSmall); EXPECT_EQ(0u, outputShapesSmall.size()); EXPECT_TRUE(badTiming(timingSmall)); diff --git a/neuralnetworks/1.2/vts/functional/ValidateRequest.cpp b/neuralnetworks/1.2/vts/functional/ValidateRequest.cpp index 7b5ff9b8e4..8498cb041b 100644 --- a/neuralnetworks/1.2/vts/functional/ValidateRequest.cpp +++ b/neuralnetworks/1.2/vts/functional/ValidateRequest.cpp @@ -22,7 +22,6 @@ #include "ExecutionBurstController.h" #include "GeneratedTestHarness.h" #include "TestHarness.h" -#include "Utils.h" #include "VtsHalNeuralnetworks.h" namespace android::hardware::neuralnetworks::V1_2::vts::functional { @@ -107,7 +106,7 @@ static void validate(const sp& preparedModel, const std::string& // execute and verify const auto [n, outputShapes, timing, fallback] = burst->compute(request, measure, keys); - const ErrorStatus status = nn::convertToV1_0(nn::convertResultCodeToErrorStatus(n)); + const ErrorStatus status = nn::legacyConvertResultCodeToErrorStatus(n); EXPECT_EQ(ErrorStatus::INVALID_ARGUMENT, status); EXPECT_EQ(outputShapes.size(), 0); EXPECT_TRUE(badTiming(timing)); diff --git a/neuralnetworks/1.3/vts/functional/ValidateBurst.cpp b/neuralnetworks/1.3/vts/functional/ValidateBurst.cpp index 6ff9dfd3a8..aecb7b79c4 100644 --- a/neuralnetworks/1.3/vts/functional/ValidateBurst.cpp +++ b/neuralnetworks/1.3/vts/functional/ValidateBurst.cpp @@ -23,7 +23,6 @@ #include "ExecutionBurstServer.h" #include "GeneratedTestHarness.h" #include "TestHarness.h" -#include "Utils.h" #include #include @@ -302,8 +301,7 @@ static void validateBurstFmqLength(const sp& preparedModel, // collect serialized result by running regular burst const auto [nRegular, outputShapesRegular, timingRegular, fallbackRegular] = controllerRegular->compute(request, MeasureTiming::NO, keys); - const V1_0::ErrorStatus statusRegular = - nn::convertToV1_0(nn::convertResultCodeToErrorStatus(nRegular)); + const V1_0::ErrorStatus statusRegular = nn::legacyConvertResultCodeToErrorStatus(nRegular); EXPECT_FALSE(fallbackRegular); // skip test if regular burst output isn't useful for testing a failure @@ -319,8 +317,7 @@ static void validateBurstFmqLength(const sp& preparedModel, // large enough to return the serialized result const auto [nSmall, outputShapesSmall, timingSmall, fallbackSmall] = controllerSmall->compute(request, MeasureTiming::NO, keys); - const V1_0::ErrorStatus statusSmall = - nn::convertToV1_0(nn::convertResultCodeToErrorStatus(nSmall)); + const V1_0::ErrorStatus statusSmall = nn::legacyConvertResultCodeToErrorStatus(nSmall); EXPECT_NE(V1_0::ErrorStatus::NONE, statusSmall); EXPECT_EQ(0u, outputShapesSmall.size()); EXPECT_TRUE(badTiming(timingSmall));