From 5f5791318422c3bc14cae9c6b456d85f6cf654ff Mon Sep 17 00:00:00 2001 From: Michael Butler Date: Thu, 1 Feb 2018 13:49:49 -0800 Subject: [PATCH 1/6] Add README to NeuralNetworks HAL Adds README explaining where to find the NeuralNetworks sample driver. Test: N/A Merged-In: Ifbbbbb3801176da27fb03bbed0b0d739031a195b Change-Id: Ifbbbbb3801176da27fb03bbed0b0d739031a195b (cherry picked from commit 3530f9ef0dbda90309ddfc626bc3d363e32dcb99) --- neuralnetworks/README | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 neuralnetworks/README diff --git a/neuralnetworks/README b/neuralnetworks/README new file mode 100644 index 0000000000..d8c8f5d2f6 --- /dev/null +++ b/neuralnetworks/README @@ -0,0 +1,2 @@ +NeuralNetworks sample driver implementation is located at +frameworks/ml/nn/driver/sample. From 4862d612add0bb302bbd2651df3cbb94f15df7ae Mon Sep 17 00:00:00 2001 From: Miao Wang Date: Mon, 5 Feb 2018 17:26:54 -0800 Subject: [PATCH 2/6] Refactor NN API VTS tests and add v1.1 tests - Create VtsHalNeuralnetworksTest_utils module to be shared between v1.x tests. - Split the existing tests into two categories: basic, and generated. - Created v1.1 VTS tests ensuring no regression in existing ML models. Bug: 63911257 Test: mm Test: NNAPI VTS tests pass on v1.0 and v1.1 sample drivers Merged-In: Ic77c90a3a5bbd96b0ce2acd03764dde4b3034cc9 Change-Id: Ic77c90a3a5bbd96b0ce2acd03764dde4b3034cc9 (cherry picked from commit a2d04c828e98bdadc6dd44c6235556451e4e2a88) --- neuralnetworks/1.0/vts/functional/Android.bp | 35 ++- .../vts/functional/GeneratedTestHarness.cpp | 167 +++++++---- neuralnetworks/1.0/vts/functional/Models.cpp | 53 ++-- neuralnetworks/1.0/vts/functional/Models.h | 27 +- .../functional/VtsHalNeuralnetworksV1_0.cpp | 73 +++++ ...argetTest.h => VtsHalNeuralnetworksV1_0.h} | 34 +-- ... => VtsHalNeuralnetworksV1_0BasicTest.cpp} | 87 ++---- .../VtsHalNeuralnetworksV1_0GeneratedTest.cpp | 72 +++++ neuralnetworks/1.1/vts/OWNERS | 10 + neuralnetworks/1.1/vts/functional/Android.bp | 39 +++ .../functional/VtsHalNeuralnetworksV1_1.cpp | 74 +++++ .../vts/functional/VtsHalNeuralnetworksV1_1.h | 85 ++++++ .../VtsHalNeuralnetworksV1_1BasicTest.cpp | 280 ++++++++++++++++++ .../VtsHalNeuralnetworksV1_1GeneratedTest.cpp | 80 +++++ 14 files changed, 929 insertions(+), 187 deletions(-) create mode 100644 neuralnetworks/1.0/vts/functional/VtsHalNeuralnetworksV1_0.cpp rename neuralnetworks/1.0/vts/functional/{VtsHalNeuralnetworksV1_0TargetTest.h => VtsHalNeuralnetworksV1_0.h} (69%) rename neuralnetworks/1.0/vts/functional/{VtsHalNeuralnetworksV1_0TargetTest.cpp => VtsHalNeuralnetworksV1_0BasicTest.cpp} (80%) create mode 100644 neuralnetworks/1.0/vts/functional/VtsHalNeuralnetworksV1_0GeneratedTest.cpp create mode 100644 neuralnetworks/1.1/vts/OWNERS create mode 100644 neuralnetworks/1.1/vts/functional/Android.bp create mode 100644 neuralnetworks/1.1/vts/functional/VtsHalNeuralnetworksV1_1.cpp create mode 100644 neuralnetworks/1.1/vts/functional/VtsHalNeuralnetworksV1_1.h create mode 100644 neuralnetworks/1.1/vts/functional/VtsHalNeuralnetworksV1_1BasicTest.cpp create mode 100644 neuralnetworks/1.1/vts/functional/VtsHalNeuralnetworksV1_1GeneratedTest.cpp diff --git a/neuralnetworks/1.0/vts/functional/Android.bp b/neuralnetworks/1.0/vts/functional/Android.bp index e33ee77f12..54dd14aba3 100644 --- a/neuralnetworks/1.0/vts/functional/Android.bp +++ b/neuralnetworks/1.0/vts/functional/Android.bp @@ -14,22 +14,49 @@ // limitations under the License. // -cc_test { - name: "VtsHalNeuralnetworksV1_0TargetTest", +cc_library_static { + name: "VtsHalNeuralnetworksTest_utils", srcs: [ "Callbacks.cpp", - "GeneratedTestHarness.cpp", "Models.cpp", - "VtsHalNeuralnetworksV1_0TargetTest.cpp", + "GeneratedTestHarness.cpp", ], defaults: ["VtsHalTargetTestDefaults"], + export_include_dirs: ["."], static_libs: [ "android.hardware.neuralnetworks@1.0", + "android.hardware.neuralnetworks@1.1", "android.hidl.allocator@1.0", "android.hidl.memory@1.0", "libhidlmemory", + "libneuralnetworks_utils", ], header_libs: [ + "libneuralnetworks_headers", + "libneuralnetworks_generated_test_harness_headers", + "libneuralnetworks_generated_tests", + ], +} + +cc_test { + name: "VtsHalNeuralnetworksV1_0TargetTest", + srcs: [ + "VtsHalNeuralnetworksV1_0.cpp", + "VtsHalNeuralnetworksV1_0BasicTest.cpp", + "VtsHalNeuralnetworksV1_0GeneratedTest.cpp", + ], + defaults: ["VtsHalTargetTestDefaults"], + static_libs: [ + "android.hardware.neuralnetworks@1.0", + "android.hardware.neuralnetworks@1.1", + "android.hidl.allocator@1.0", + "android.hidl.memory@1.0", + "libhidlmemory", + "libneuralnetworks_utils", + "VtsHalNeuralnetworksTest_utils", + ], + header_libs: [ + "libneuralnetworks_headers", "libneuralnetworks_generated_test_harness_headers", "libneuralnetworks_generated_tests", ], diff --git a/neuralnetworks/1.0/vts/functional/GeneratedTestHarness.cpp b/neuralnetworks/1.0/vts/functional/GeneratedTestHarness.cpp index d740b5f53c..5fe8415147 100644 --- a/neuralnetworks/1.0/vts/functional/GeneratedTestHarness.cpp +++ b/neuralnetworks/1.0/vts/functional/GeneratedTestHarness.cpp @@ -16,9 +16,15 @@ #include "Callbacks.h" #include "TestHarness.h" -#include "VtsHalNeuralnetworksV1_0TargetTest.h" +#include "Utils.h" #include +#include +#include +#include +#include +#include +#include #include #include #include @@ -26,11 +32,6 @@ namespace android { namespace hardware { namespace neuralnetworks { -namespace V1_0 { -namespace vts { -namespace functional { -// allocator helper -hidl_memory allocateSharedMemory(int64_t size, const std::string& type = "ashmem"); namespace generated_tests { using ::android::hardware::neuralnetworks::V1_0::implementation::ExecutionCallback; @@ -64,54 +65,10 @@ void copy_back(MixedTyped* dst, const std::vector& ra, char* sr // Top level driver for models and examples generated by test_generator.py // Test driver for those generated from ml/nn/runtime/test/spec -void Execute(const sp& device, std::function create_model, - std::function is_ignored, - const std::vector& examples) { +void EvaluatePreparedModel(sp& preparedModel, std::function is_ignored, + const std::vector& examples) { const uint32_t INPUT = 0; const uint32_t OUTPUT = 1; - Model model = create_model(); - - // see if service can handle model - ErrorStatus supportedStatus; - bool fullySupportsModel = false; - Return supportedCall = device->getSupportedOperations( - model, [&](ErrorStatus status, const hidl_vec& supported) { - supportedStatus = status; - ASSERT_NE(0ul, supported.size()); - fullySupportsModel = - std::all_of(supported.begin(), supported.end(), [](bool valid) { return valid; }); - }); - ASSERT_TRUE(supportedCall.isOk()); - ASSERT_EQ(ErrorStatus::NONE, supportedStatus); - - // launch prepare model - sp preparedModelCallback = new PreparedModelCallback(); - ASSERT_NE(nullptr, preparedModelCallback.get()); - Return prepareLaunchStatus = device->prepareModel(model, preparedModelCallback); - ASSERT_TRUE(prepareLaunchStatus.isOk()); - - // retrieve prepared model - preparedModelCallback->wait(); - ErrorStatus prepareReturnStatus = preparedModelCallback->getStatus(); - sp preparedModel = preparedModelCallback->getPreparedModel(); - if (fullySupportsModel) { - EXPECT_EQ(ErrorStatus::NONE, prepareReturnStatus); - } else { - EXPECT_TRUE(prepareReturnStatus == ErrorStatus::NONE || - prepareReturnStatus == ErrorStatus::GENERAL_FAILURE); - } - - // early termination if vendor service cannot fully prepare model - if (!fullySupportsModel && prepareReturnStatus == ErrorStatus::GENERAL_FAILURE) { - ASSERT_EQ(nullptr, preparedModel.get()); - LOG(INFO) << "NN VTS: Early termination of test because vendor service cannot " - "prepare model that it does not support."; - std::cout << "[ ] Early termination of test because vendor service cannot " - "prepare model that it does not support." - << std::endl; - return; - } - ASSERT_NE(nullptr, preparedModel.get()); int example_no = 1; for (auto& example : examples) { @@ -167,8 +124,8 @@ void Execute(const sp& device, std::function create_model, offset += i.location.length; } } - std::vector pools = {allocateSharedMemory(inputSize), - allocateSharedMemory(outputSize)}; + std::vector pools = {nn::allocateSharedMemory(inputSize), + nn::allocateSharedMemory(outputSize)}; ASSERT_NE(0ull, pools[INPUT].size()); ASSERT_NE(0ull, pools[OUTPUT].size()); @@ -221,11 +178,107 @@ void Execute(const sp& device, std::function create_model, } } +void Execute(sp& device, std::function create_model, + std::function is_ignored, + const std::vector& examples) { + V1_0::Model model = create_model(); + + // see if service can handle model + bool fullySupportsModel = false; + ErrorStatus supportedStatus; + sp preparedModelCallback = new PreparedModelCallback(); + ASSERT_NE(nullptr, preparedModelCallback.get()); + + Return supportedCall = device->getSupportedOperations( + model, [&](ErrorStatus status, const hidl_vec& supported) { + supportedStatus = status; + ASSERT_NE(0ul, supported.size()); + fullySupportsModel = + std::all_of(supported.begin(), supported.end(), [](bool valid) { return valid; }); + }); + ASSERT_TRUE(supportedCall.isOk()); + ASSERT_EQ(ErrorStatus::NONE, supportedStatus); + Return prepareLaunchStatus = device->prepareModel(model, preparedModelCallback); + ASSERT_TRUE(prepareLaunchStatus.isOk()); + + // retrieve prepared model + preparedModelCallback->wait(); + ErrorStatus prepareReturnStatus = preparedModelCallback->getStatus(); + sp preparedModel = preparedModelCallback->getPreparedModel(); + if (fullySupportsModel) { + EXPECT_EQ(ErrorStatus::NONE, prepareReturnStatus); + } else { + EXPECT_TRUE(prepareReturnStatus == ErrorStatus::NONE || + prepareReturnStatus == ErrorStatus::GENERAL_FAILURE); + } + + // early termination if vendor service cannot fully prepare model + if (!fullySupportsModel && prepareReturnStatus == ErrorStatus::GENERAL_FAILURE) { + ASSERT_EQ(nullptr, preparedModel.get()); + LOG(INFO) << "NN VTS: Early termination of test because vendor service cannot " + "prepare model that it does not support."; + std::cout << "[ ] Early termination of test because vendor service cannot " + "prepare model that it does not support." + << std::endl; + return; + } + ASSERT_NE(nullptr, preparedModel.get()); + + EvaluatePreparedModel(preparedModel, is_ignored, examples); +} + +void Execute(sp& device, std::function create_model, + std::function is_ignored, + const std::vector& examples) { + V1_1::Model model = create_model(); + + // see if service can handle model + bool fullySupportsModel = false; + ErrorStatus supportedStatus; + sp preparedModelCallback = new PreparedModelCallback(); + ASSERT_NE(nullptr, preparedModelCallback.get()); + + Return supportedCall = device->getSupportedOperations_1_1( + model, [&](ErrorStatus status, const hidl_vec& supported) { + supportedStatus = status; + ASSERT_NE(0ul, supported.size()); + fullySupportsModel = + std::all_of(supported.begin(), supported.end(), [](bool valid) { return valid; }); + }); + ASSERT_TRUE(supportedCall.isOk()); + ASSERT_EQ(ErrorStatus::NONE, supportedStatus); + Return prepareLaunchStatus = + device->prepareModel_1_1(model, preparedModelCallback); + ASSERT_TRUE(prepareLaunchStatus.isOk()); + + // retrieve prepared model + preparedModelCallback->wait(); + ErrorStatus prepareReturnStatus = preparedModelCallback->getStatus(); + sp preparedModel = preparedModelCallback->getPreparedModel(); + if (fullySupportsModel) { + EXPECT_EQ(ErrorStatus::NONE, prepareReturnStatus); + } else { + EXPECT_TRUE(prepareReturnStatus == ErrorStatus::NONE || + prepareReturnStatus == ErrorStatus::GENERAL_FAILURE); + } + + // early termination if vendor service cannot fully prepare model + if (!fullySupportsModel && prepareReturnStatus == ErrorStatus::GENERAL_FAILURE) { + ASSERT_EQ(nullptr, preparedModel.get()); + LOG(INFO) << "NN VTS: Early termination of test because vendor service cannot " + "prepare model that it does not support."; + std::cout << "[ ] Early termination of test because vendor service cannot " + "prepare model that it does not support." + << std::endl; + return; + } + ASSERT_NE(nullptr, preparedModel.get()); + + EvaluatePreparedModel(preparedModel, is_ignored, examples); +} + } // namespace generated_tests -} // namespace functional -} // namespace vts -} // namespace V1_0 } // namespace neuralnetworks } // namespace hardware } // namespace android diff --git a/neuralnetworks/1.0/vts/functional/Models.cpp b/neuralnetworks/1.0/vts/functional/Models.cpp index 8ce4f25938..180286a5b7 100644 --- a/neuralnetworks/1.0/vts/functional/Models.cpp +++ b/neuralnetworks/1.0/vts/functional/Models.cpp @@ -17,19 +17,22 @@ #define LOG_TAG "neuralnetworks_hidl_hal_test" #include "Models.h" +#include "Utils.h" + +#include +#include #include #include #include +using ::android::sp; + namespace android { namespace hardware { namespace neuralnetworks { -namespace V1_0 { -namespace vts { -namespace functional { // create a valid model -Model createValidTestModel() { +V1_1::Model createValidTestModel_1_1() { const std::vector operand2Data = {5.0f, 6.0f, 7.0f, 8.0f}; const uint32_t size = operand2Data.size() * sizeof(float); @@ -103,39 +106,34 @@ Model createValidTestModel() { } // create first invalid model -Model createInvalidTestModel1() { - Model model = createValidTestModel(); +V1_1::Model createInvalidTestModel1_1_1() { + Model model = createValidTestModel_1_1(); model.operations[0].type = static_cast(0xDEADBEEF); /* INVALID */ return model; } // create second invalid model -Model createInvalidTestModel2() { - Model model = createValidTestModel(); +V1_1::Model createInvalidTestModel2_1_1() { + Model model = createValidTestModel_1_1(); const uint32_t operand1 = 0; const uint32_t operand5 = 4; // INVALID OPERAND model.inputIndexes = std::vector({operand1, operand5 /* INVALID OPERAND */}); return model; } -// allocator helper -hidl_memory allocateSharedMemory(int64_t size, const std::string& type = "ashmem") { - hidl_memory memory; +V1_0::Model createValidTestModel_1_0() { + V1_1::Model model = createValidTestModel_1_1(); + return nn::convertToV1_0(model); +} - sp allocator = IAllocator::getService(type); - if (!allocator.get()) { - return {}; - } +V1_0::Model createInvalidTestModel1_1_0() { + V1_1::Model model = createInvalidTestModel1_1_1(); + return nn::convertToV1_0(model); +} - Return ret = allocator->allocate(size, [&](bool success, const hidl_memory& mem) { - ASSERT_TRUE(success); - memory = mem; - }); - if (!ret.isOk()) { - return {}; - } - - return memory; +V1_0::Model createInvalidTestModel2_1_0() { + V1_1::Model model = createInvalidTestModel2_1_1(); + return nn::convertToV1_0(model); } // create a valid request @@ -154,8 +152,8 @@ Request createValidTestRequest() { std::vector outputs = {{ .location = {.poolIndex = OUTPUT, .offset = 0, .length = outputSize}, .dimensions = {}, }}; - std::vector pools = {allocateSharedMemory(inputSize), - allocateSharedMemory(outputSize)}; + std::vector pools = {nn::allocateSharedMemory(inputSize), + nn::allocateSharedMemory(outputSize)}; if (pools[INPUT].size() == 0 || pools[OUTPUT].size() == 0) { return {}; } @@ -199,9 +197,6 @@ Request createInvalidTestRequest2() { return request; } -} // namespace functional -} // namespace vts -} // namespace V1_0 } // namespace neuralnetworks } // namespace hardware } // namespace android diff --git a/neuralnetworks/1.0/vts/functional/Models.h b/neuralnetworks/1.0/vts/functional/Models.h index e0d57d533b..93982351f4 100644 --- a/neuralnetworks/1.0/vts/functional/Models.h +++ b/neuralnetworks/1.0/vts/functional/Models.h @@ -16,28 +16,27 @@ #define LOG_TAG "neuralnetworks_hidl_hal_test" -#include "VtsHalNeuralnetworksV1_0TargetTest.h" +#include namespace android { namespace hardware { namespace neuralnetworks { -namespace V1_0 { -namespace vts { -namespace functional { -// create the model -Model createValidTestModel(); -Model createInvalidTestModel1(); -Model createInvalidTestModel2(); +// create V1_1 model +V1_1::Model createValidTestModel_1_1(); +V1_1::Model createInvalidTestModel1_1_1(); +V1_1::Model createInvalidTestModel2_1_1(); + +// create V1_0 model +V1_0::Model createValidTestModel_1_0(); +V1_0::Model createInvalidTestModel1_1_0(); +V1_0::Model createInvalidTestModel2_1_0(); // create the request -Request createValidTestRequest(); -Request createInvalidTestRequest1(); -Request createInvalidTestRequest2(); +V1_0::Request createValidTestRequest(); +V1_0::Request createInvalidTestRequest1(); +V1_0::Request createInvalidTestRequest2(); -} // namespace functional -} // namespace vts -} // namespace V1_0 } // namespace neuralnetworks } // namespace hardware } // namespace android diff --git a/neuralnetworks/1.0/vts/functional/VtsHalNeuralnetworksV1_0.cpp b/neuralnetworks/1.0/vts/functional/VtsHalNeuralnetworksV1_0.cpp new file mode 100644 index 0000000000..b14fb2c4c8 --- /dev/null +++ b/neuralnetworks/1.0/vts/functional/VtsHalNeuralnetworksV1_0.cpp @@ -0,0 +1,73 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#define LOG_TAG "neuralnetworks_hidl_hal_test" + +#include "VtsHalNeuralnetworksV1_0.h" +#include "Utils.h" + +#include + +using ::android::hardware::hidl_memory; +using ::android::hidl::allocator::V1_0::IAllocator; +using ::android::hidl::memory::V1_0::IMemory; +using ::android::sp; + +namespace android { +namespace hardware { +namespace neuralnetworks { +namespace V1_0 { +namespace vts { +namespace functional { + +// allocator helper +hidl_memory allocateSharedMemory(int64_t size) { + return nn::allocateSharedMemory(size); +} + +// A class for test environment setup +NeuralnetworksHidlEnvironment::NeuralnetworksHidlEnvironment() {} + +NeuralnetworksHidlEnvironment::~NeuralnetworksHidlEnvironment() {} + +NeuralnetworksHidlEnvironment* NeuralnetworksHidlEnvironment::getInstance() { + // This has to return a "new" object because it is freed inside + // ::testing::AddGlobalTestEnvironment when the gtest is being torn down + static NeuralnetworksHidlEnvironment* instance = new NeuralnetworksHidlEnvironment(); + return instance; +} + +void NeuralnetworksHidlEnvironment::registerTestServices() { + registerTestService(); +} + +// The main test class for NEURALNETWORK HIDL HAL. +NeuralnetworksHidlTest::~NeuralnetworksHidlTest() {} + +void NeuralnetworksHidlTest::SetUp() { + device = ::testing::VtsHalHidlTargetTestBase::getService( + NeuralnetworksHidlEnvironment::getInstance()); + ASSERT_NE(nullptr, device.get()); +} + +void NeuralnetworksHidlTest::TearDown() {} + +} // namespace functional +} // namespace vts +} // namespace V1_0 +} // namespace neuralnetworks +} // namespace hardware +} // namespace android diff --git a/neuralnetworks/1.0/vts/functional/VtsHalNeuralnetworksV1_0TargetTest.h b/neuralnetworks/1.0/vts/functional/VtsHalNeuralnetworksV1_0.h similarity index 69% rename from neuralnetworks/1.0/vts/functional/VtsHalNeuralnetworksV1_0TargetTest.h rename to neuralnetworks/1.0/vts/functional/VtsHalNeuralnetworksV1_0.h index 5cd209ae62..fbb1607478 100644 --- a/neuralnetworks/1.0/vts/functional/VtsHalNeuralnetworksV1_0TargetTest.h +++ b/neuralnetworks/1.0/vts/functional/VtsHalNeuralnetworksV1_0.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2017 The Android Open Source Project + * Copyright (C) 2018 The Android Open Source Project * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -29,23 +29,6 @@ #include #include -using ::android::hardware::neuralnetworks::V1_0::IDevice; -using ::android::hardware::neuralnetworks::V1_0::IPreparedModel; -using ::android::hardware::neuralnetworks::V1_0::Capabilities; -using ::android::hardware::neuralnetworks::V1_0::DeviceStatus; -using ::android::hardware::neuralnetworks::V1_0::FusedActivationFunc; -using ::android::hardware::neuralnetworks::V1_0::Model; -using ::android::hardware::neuralnetworks::V1_0::OperationType; -using ::android::hardware::neuralnetworks::V1_0::PerformanceInfo; -using ::android::hardware::Return; -using ::android::hardware::Void; -using ::android::hardware::hidl_memory; -using ::android::hardware::hidl_string; -using ::android::hardware::hidl_vec; -using ::android::hidl::allocator::V1_0::IAllocator; -using ::android::hidl::memory::V1_0::IMemory; -using ::android::sp; - namespace android { namespace hardware { namespace neuralnetworks { @@ -53,6 +36,8 @@ namespace V1_0 { namespace vts { namespace functional { +hidl_memory allocateSharedMemory(int64_t size); + // A class for test environment setup class NeuralnetworksHidlEnvironment : public ::testing::VtsHalHidlTargetTestEnvBase { NeuralnetworksHidlEnvironment(); @@ -74,25 +59,22 @@ class NeuralnetworksHidlTest : public ::testing::VtsHalHidlTargetTestBase { void SetUp() override; void TearDown() override; - sp doPrepareModelShortcut(); - - sp device; + sp device; }; - } // namespace functional } // namespace vts // pretty-print values for error messages -template +template ::std::basic_ostream& operator<<(::std::basic_ostream& os, - ErrorStatus errorStatus) { + V1_0::ErrorStatus errorStatus) { return os << toString(errorStatus); } -template +template ::std::basic_ostream& operator<<(::std::basic_ostream& os, - DeviceStatus deviceStatus) { + V1_0::DeviceStatus deviceStatus) { return os << toString(deviceStatus); } diff --git a/neuralnetworks/1.0/vts/functional/VtsHalNeuralnetworksV1_0TargetTest.cpp b/neuralnetworks/1.0/vts/functional/VtsHalNeuralnetworksV1_0BasicTest.cpp similarity index 80% rename from neuralnetworks/1.0/vts/functional/VtsHalNeuralnetworksV1_0TargetTest.cpp rename to neuralnetworks/1.0/vts/functional/VtsHalNeuralnetworksV1_0BasicTest.cpp index b99e20e3b4..e838997a8d 100644 --- a/neuralnetworks/1.0/vts/functional/VtsHalNeuralnetworksV1_0TargetTest.cpp +++ b/neuralnetworks/1.0/vts/functional/VtsHalNeuralnetworksV1_0BasicTest.cpp @@ -1,5 +1,5 @@ /* - * Copyright (C) 2017 The Android Open Source Project + * Copyright (C) 2018 The Android Open Source Project * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,7 +16,7 @@ #define LOG_TAG "neuralnetworks_hidl_hal_test" -#include "VtsHalNeuralnetworksV1_0TargetTest.h" +#include "VtsHalNeuralnetworksV1_0.h" #include "Callbacks.h" #include "Models.h" @@ -26,51 +26,34 @@ #include #include +using ::android::hardware::neuralnetworks::V1_0::IDevice; +using ::android::hardware::neuralnetworks::V1_0::IPreparedModel; +using ::android::hardware::neuralnetworks::V1_0::Capabilities; +using ::android::hardware::neuralnetworks::V1_0::DeviceStatus; +using ::android::hardware::neuralnetworks::V1_0::FusedActivationFunc; +using ::android::hardware::neuralnetworks::V1_0::Model; +using ::android::hardware::neuralnetworks::V1_0::OperationType; +using ::android::hardware::neuralnetworks::V1_0::PerformanceInfo; +using ::android::hardware::Return; +using ::android::hardware::Void; +using ::android::hardware::hidl_memory; +using ::android::hardware::hidl_string; +using ::android::hardware::hidl_vec; +using ::android::hidl::allocator::V1_0::IAllocator; +using ::android::hidl::memory::V1_0::IMemory; +using ::android::sp; + namespace android { namespace hardware { namespace neuralnetworks { namespace V1_0 { namespace vts { namespace functional { - using ::android::hardware::neuralnetworks::V1_0::implementation::ExecutionCallback; using ::android::hardware::neuralnetworks::V1_0::implementation::PreparedModelCallback; -using ::generated_tests::MixedTypedExampleType; -namespace generated_tests { -extern void Execute(const sp&, std::function, std::function, - const std::vector&); -} - -// A class for test environment setup -NeuralnetworksHidlEnvironment::NeuralnetworksHidlEnvironment() {} - -NeuralnetworksHidlEnvironment::~NeuralnetworksHidlEnvironment() {} - -NeuralnetworksHidlEnvironment* NeuralnetworksHidlEnvironment::getInstance() { - // This has to return a "new" object because it is freed inside - // ::testing::AddGlobalTestEnvironment when the gtest is being torn down - static NeuralnetworksHidlEnvironment* instance = new NeuralnetworksHidlEnvironment(); - return instance; -} - -void NeuralnetworksHidlEnvironment::registerTestServices() { - registerTestService(); -} - -// The main test class for NEURALNETWORK HIDL HAL. -NeuralnetworksHidlTest::~NeuralnetworksHidlTest() {} - -void NeuralnetworksHidlTest::SetUp() { - device = ::testing::VtsHalHidlTargetTestBase::getService( - NeuralnetworksHidlEnvironment::getInstance()); - ASSERT_NE(nullptr, device.get()); -} - -void NeuralnetworksHidlTest::TearDown() {} - -sp NeuralnetworksHidlTest::doPrepareModelShortcut() { - Model model = createValidTestModel(); +inline sp doPrepareModelShortcut(sp& device) { + Model model = createValidTestModel_1_0(); sp preparedModelCallback = new PreparedModelCallback(); if (preparedModelCallback == nullptr) { @@ -116,7 +99,7 @@ TEST_F(NeuralnetworksHidlTest, GetCapabilitiesTest) { // supported operations positive test TEST_F(NeuralnetworksHidlTest, SupportedOperationsPositiveTest) { - Model model = createValidTestModel(); + Model model = createValidTestModel_1_0(); Return ret = device->getSupportedOperations( model, [&](ErrorStatus status, const hidl_vec& supported) { EXPECT_EQ(ErrorStatus::NONE, status); @@ -127,7 +110,7 @@ TEST_F(NeuralnetworksHidlTest, SupportedOperationsPositiveTest) { // supported operations negative test 1 TEST_F(NeuralnetworksHidlTest, SupportedOperationsNegativeTest1) { - Model model = createInvalidTestModel1(); + Model model = createInvalidTestModel1_1_0(); Return ret = device->getSupportedOperations( model, [&](ErrorStatus status, const hidl_vec& supported) { EXPECT_EQ(ErrorStatus::INVALID_ARGUMENT, status); @@ -138,7 +121,7 @@ TEST_F(NeuralnetworksHidlTest, SupportedOperationsNegativeTest1) { // supported operations negative test 2 TEST_F(NeuralnetworksHidlTest, SupportedOperationsNegativeTest2) { - Model model = createInvalidTestModel2(); + Model model = createInvalidTestModel2_1_0(); Return ret = device->getSupportedOperations( model, [&](ErrorStatus status, const hidl_vec& supported) { EXPECT_EQ(ErrorStatus::INVALID_ARGUMENT, status); @@ -149,7 +132,7 @@ TEST_F(NeuralnetworksHidlTest, SupportedOperationsNegativeTest2) { // prepare simple model positive test TEST_F(NeuralnetworksHidlTest, SimplePrepareModelPositiveTest) { - Model model = createValidTestModel(); + Model model = createValidTestModel_1_0(); sp preparedModelCallback = new PreparedModelCallback(); ASSERT_NE(nullptr, preparedModelCallback.get()); Return prepareLaunchStatus = device->prepareModel(model, preparedModelCallback); @@ -165,7 +148,7 @@ TEST_F(NeuralnetworksHidlTest, SimplePrepareModelPositiveTest) { // prepare simple model negative test 1 TEST_F(NeuralnetworksHidlTest, SimplePrepareModelNegativeTest1) { - Model model = createInvalidTestModel1(); + Model model = createInvalidTestModel1_1_0(); sp preparedModelCallback = new PreparedModelCallback(); ASSERT_NE(nullptr, preparedModelCallback.get()); Return prepareLaunchStatus = device->prepareModel(model, preparedModelCallback); @@ -181,7 +164,7 @@ TEST_F(NeuralnetworksHidlTest, SimplePrepareModelNegativeTest1) { // prepare simple model negative test 2 TEST_F(NeuralnetworksHidlTest, SimplePrepareModelNegativeTest2) { - Model model = createInvalidTestModel2(); + Model model = createInvalidTestModel2_1_0(); sp preparedModelCallback = new PreparedModelCallback(); ASSERT_NE(nullptr, preparedModelCallback.get()); Return prepareLaunchStatus = device->prepareModel(model, preparedModelCallback); @@ -201,7 +184,7 @@ TEST_F(NeuralnetworksHidlTest, SimpleExecuteGraphPositiveTest) { std::vector expectedData = {6.0f, 8.0f, 10.0f, 12.0f}; const uint32_t OUTPUT = 1; - sp preparedModel = doPrepareModelShortcut(); + sp preparedModel = doPrepareModelShortcut(device); ASSERT_NE(nullptr, preparedModel.get()); Request request = createValidTestRequest(); @@ -235,7 +218,7 @@ TEST_F(NeuralnetworksHidlTest, SimpleExecuteGraphPositiveTest) { // execute simple graph negative test 1 TEST_F(NeuralnetworksHidlTest, SimpleExecuteGraphNegativeTest1) { - sp preparedModel = doPrepareModelShortcut(); + sp preparedModel = doPrepareModelShortcut(device); ASSERT_NE(nullptr, preparedModel.get()); Request request = createInvalidTestRequest1(); @@ -252,7 +235,7 @@ TEST_F(NeuralnetworksHidlTest, SimpleExecuteGraphNegativeTest1) { // execute simple graph negative test 2 TEST_F(NeuralnetworksHidlTest, SimpleExecuteGraphNegativeTest2) { - sp preparedModel = doPrepareModelShortcut(); + sp preparedModel = doPrepareModelShortcut(device); ASSERT_NE(nullptr, preparedModel.get()); Request request = createInvalidTestRequest2(); @@ -267,16 +250,6 @@ TEST_F(NeuralnetworksHidlTest, SimpleExecuteGraphNegativeTest2) { EXPECT_EQ(ErrorStatus::INVALID_ARGUMENT, executionReturnStatus); } -// Mixed-typed examples -typedef MixedTypedExampleType MixedTypedExample; - -// in frameworks/ml/nn/runtime/tests/generated/ -#include "all_generated_vts_tests.cpp" - -// TODO: Add tests for execution failure, or wait_for/wait_until timeout. -// Discussion: -// https://googleplex-android-review.git.corp.google.com/#/c/platform/hardware/interfaces/+/2654636/5/neuralnetworks/1.0/vts/functional/VtsHalNeuralnetworksV1_0TargetTest.cpp@222 - } // namespace functional } // namespace vts } // namespace V1_0 diff --git a/neuralnetworks/1.0/vts/functional/VtsHalNeuralnetworksV1_0GeneratedTest.cpp b/neuralnetworks/1.0/vts/functional/VtsHalNeuralnetworksV1_0GeneratedTest.cpp new file mode 100644 index 0000000000..b99aef7fc0 --- /dev/null +++ b/neuralnetworks/1.0/vts/functional/VtsHalNeuralnetworksV1_0GeneratedTest.cpp @@ -0,0 +1,72 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#define LOG_TAG "neuralnetworks_hidl_hal_test" + +#include "VtsHalNeuralnetworksV1_0.h" + +#include "Callbacks.h" +#include "TestHarness.h" + +#include +#include +#include + +using ::android::hardware::neuralnetworks::V1_0::IDevice; +using ::android::hardware::neuralnetworks::V1_0::IPreparedModel; +using ::android::hardware::neuralnetworks::V1_0::Capabilities; +using ::android::hardware::neuralnetworks::V1_0::DeviceStatus; +using ::android::hardware::neuralnetworks::V1_0::FusedActivationFunc; +using ::android::hardware::neuralnetworks::V1_0::Model; +using ::android::hardware::neuralnetworks::V1_0::OperationType; +using ::android::hardware::neuralnetworks::V1_0::PerformanceInfo; +using ::android::hardware::Return; +using ::android::hardware::Void; +using ::android::hardware::hidl_memory; +using ::android::hardware::hidl_string; +using ::android::hardware::hidl_vec; +using ::android::hidl::allocator::V1_0::IAllocator; +using ::android::hidl::memory::V1_0::IMemory; +using ::android::sp; + +namespace android { +namespace hardware { +namespace neuralnetworks { + +namespace generated_tests { +using ::generated_tests::MixedTypedExampleType; +extern void Execute(sp&, std::function, std::function, + const std::vector&); +} // namespace generated_tests + +namespace V1_0 { +namespace vts { +namespace functional { +using ::android::hardware::neuralnetworks::V1_0::implementation::ExecutionCallback; +using ::android::hardware::neuralnetworks::V1_0::implementation::PreparedModelCallback; + +// Mixed-typed examples +typedef generated_tests::MixedTypedExampleType MixedTypedExample; + +// in frameworks/ml/nn/runtime/tests/generated/ +#include "all_generated_V1_0_vts_tests.cpp" + +} // namespace functional +} // namespace vts +} // namespace V1_0 +} // namespace neuralnetworks +} // namespace hardware +} // namespace android diff --git a/neuralnetworks/1.1/vts/OWNERS b/neuralnetworks/1.1/vts/OWNERS new file mode 100644 index 0000000000..7f75ab30c6 --- /dev/null +++ b/neuralnetworks/1.1/vts/OWNERS @@ -0,0 +1,10 @@ +# Neuralnetworks team +butlermichael@google.com +dgross@google.com +jeanluc@google.com +miaowang@google.com +yangni@google.com + +# VTS team +yim@google.com +yuexima@google.com diff --git a/neuralnetworks/1.1/vts/functional/Android.bp b/neuralnetworks/1.1/vts/functional/Android.bp new file mode 100644 index 0000000000..623b44103a --- /dev/null +++ b/neuralnetworks/1.1/vts/functional/Android.bp @@ -0,0 +1,39 @@ +// +// Copyright (C) 2018 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +cc_test { + name: "VtsHalNeuralnetworksV1_1TargetTest", + srcs: [ + "VtsHalNeuralnetworksV1_1.cpp", + "VtsHalNeuralnetworksV1_1BasicTest.cpp", + "VtsHalNeuralnetworksV1_1GeneratedTest.cpp", + ], + defaults: ["VtsHalTargetTestDefaults"], + static_libs: [ + "android.hardware.neuralnetworks@1.0", + "android.hardware.neuralnetworks@1.1", + "android.hidl.allocator@1.0", + "android.hidl.memory@1.0", + "libhidlmemory", + "libneuralnetworks_utils", + "VtsHalNeuralnetworksTest_utils", + ], + header_libs: [ + "libneuralnetworks_headers", + "libneuralnetworks_generated_test_harness_headers", + "libneuralnetworks_generated_tests", + ], +} diff --git a/neuralnetworks/1.1/vts/functional/VtsHalNeuralnetworksV1_1.cpp b/neuralnetworks/1.1/vts/functional/VtsHalNeuralnetworksV1_1.cpp new file mode 100644 index 0000000000..b1d3be786c --- /dev/null +++ b/neuralnetworks/1.1/vts/functional/VtsHalNeuralnetworksV1_1.cpp @@ -0,0 +1,74 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#define LOG_TAG "neuralnetworks_hidl_hal_test" + +#include "VtsHalNeuralnetworksV1_1.h" +#include "Utils.h" + +#include +#include + +using ::android::hardware::hidl_memory; +using ::android::hidl::allocator::V1_0::IAllocator; +using ::android::hidl::memory::V1_0::IMemory; +using ::android::sp; + +namespace android { +namespace hardware { +namespace neuralnetworks { +namespace V1_1 { +namespace vts { +namespace functional { + +// allocator helper +hidl_memory allocateSharedMemory(int64_t size) { + return nn::allocateSharedMemory(size); +} + +// A class for test environment setup +NeuralnetworksHidlEnvironment::NeuralnetworksHidlEnvironment() {} + +NeuralnetworksHidlEnvironment::~NeuralnetworksHidlEnvironment() {} + +NeuralnetworksHidlEnvironment* NeuralnetworksHidlEnvironment::getInstance() { + // This has to return a "new" object because it is freed inside + // ::testing::AddGlobalTestEnvironment when the gtest is being torn down + static NeuralnetworksHidlEnvironment* instance = new NeuralnetworksHidlEnvironment(); + return instance; +} + +void NeuralnetworksHidlEnvironment::registerTestServices() { + registerTestService(); +} + +// The main test class for NEURALNETWORK HIDL HAL. +NeuralnetworksHidlTest::~NeuralnetworksHidlTest() {} + +void NeuralnetworksHidlTest::SetUp() { + device = ::testing::VtsHalHidlTargetTestBase::getService( + NeuralnetworksHidlEnvironment::getInstance()); + ASSERT_NE(nullptr, device.get()); +} + +void NeuralnetworksHidlTest::TearDown() {} + +} // namespace functional +} // namespace vts +} // namespace V1_1 +} // namespace neuralnetworks +} // namespace hardware +} // namespace android diff --git a/neuralnetworks/1.1/vts/functional/VtsHalNeuralnetworksV1_1.h b/neuralnetworks/1.1/vts/functional/VtsHalNeuralnetworksV1_1.h new file mode 100644 index 0000000000..426246ce76 --- /dev/null +++ b/neuralnetworks/1.1/vts/functional/VtsHalNeuralnetworksV1_1.h @@ -0,0 +1,85 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef VTS_HAL_NEURALNETWORKS_V1_1_H +#define VTS_HAL_NEURALNETWORKS_V1_1_H + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +namespace android { +namespace hardware { +namespace neuralnetworks { +namespace V1_1 { +namespace vts { +namespace functional { +hidl_memory allocateSharedMemory(int64_t size); + +// A class for test environment setup +class NeuralnetworksHidlEnvironment : public ::testing::VtsHalHidlTargetTestEnvBase { + NeuralnetworksHidlEnvironment(); + NeuralnetworksHidlEnvironment(const NeuralnetworksHidlEnvironment&) = delete; + NeuralnetworksHidlEnvironment(NeuralnetworksHidlEnvironment&&) = delete; + NeuralnetworksHidlEnvironment& operator=(const NeuralnetworksHidlEnvironment&) = delete; + NeuralnetworksHidlEnvironment& operator=(NeuralnetworksHidlEnvironment&&) = delete; + + public: + ~NeuralnetworksHidlEnvironment() override; + static NeuralnetworksHidlEnvironment* getInstance(); + void registerTestServices() override; +}; + +// The main test class for NEURALNETWORKS HIDL HAL. +class NeuralnetworksHidlTest : public ::testing::VtsHalHidlTargetTestBase { + public: + ~NeuralnetworksHidlTest() override; + void SetUp() override; + void TearDown() override; + + sp device; +}; +} // namespace functional +} // namespace vts + +// pretty-print values for error messages + +template +::std::basic_ostream& operator<<(::std::basic_ostream& os, + V1_0::ErrorStatus errorStatus) { + return os << toString(errorStatus); +} + +template +::std::basic_ostream& operator<<(::std::basic_ostream& os, + V1_0::DeviceStatus deviceStatus) { + return os << toString(deviceStatus); +} + +} // namespace V1_1 +} // namespace neuralnetworks +} // namespace hardware +} // namespace android + +#endif // VTS_HAL_NEURALNETWORKS_V1_1_H diff --git a/neuralnetworks/1.1/vts/functional/VtsHalNeuralnetworksV1_1BasicTest.cpp b/neuralnetworks/1.1/vts/functional/VtsHalNeuralnetworksV1_1BasicTest.cpp new file mode 100644 index 0000000000..51eff2a019 --- /dev/null +++ b/neuralnetworks/1.1/vts/functional/VtsHalNeuralnetworksV1_1BasicTest.cpp @@ -0,0 +1,280 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#define LOG_TAG "neuralnetworks_hidl_hal_test" + +#include "VtsHalNeuralnetworksV1_1.h" + +#include "Callbacks.h" +#include "Models.h" +#include "TestHarness.h" + +#include +#include +#include +#include +#include + +using ::android::hardware::neuralnetworks::V1_0::IPreparedModel; +using ::android::hardware::neuralnetworks::V1_0::Capabilities; +using ::android::hardware::neuralnetworks::V1_0::DeviceStatus; +using ::android::hardware::neuralnetworks::V1_0::ErrorStatus; +using ::android::hardware::neuralnetworks::V1_0::FusedActivationFunc; +using ::android::hardware::neuralnetworks::V1_0::Operand; +using ::android::hardware::neuralnetworks::V1_0::OperandLifeTime; +using ::android::hardware::neuralnetworks::V1_0::OperandType; +using ::android::hardware::neuralnetworks::V1_0::Request; +using ::android::hardware::neuralnetworks::V1_1::IDevice; +using ::android::hardware::neuralnetworks::V1_1::Model; +using ::android::hardware::neuralnetworks::V1_1::Operation; +using ::android::hardware::neuralnetworks::V1_1::OperationType; +using ::android::hardware::Return; +using ::android::hardware::Void; +using ::android::hardware::hidl_memory; +using ::android::hardware::hidl_string; +using ::android::hardware::hidl_vec; +using ::android::hidl::allocator::V1_0::IAllocator; +using ::android::hidl::memory::V1_0::IMemory; +using ::android::sp; + +namespace android { +namespace hardware { +namespace neuralnetworks { +namespace V1_1 { +namespace vts { +namespace functional { +using ::android::hardware::neuralnetworks::V1_0::implementation::ExecutionCallback; +using ::android::hardware::neuralnetworks::V1_0::implementation::PreparedModelCallback; + +inline sp doPrepareModelShortcut(sp& device) { + Model model = createValidTestModel_1_1(); + + sp preparedModelCallback = new PreparedModelCallback(); + if (preparedModelCallback == nullptr) { + return nullptr; + } + Return prepareLaunchStatus = + device->prepareModel_1_1(model, preparedModelCallback); + if (!prepareLaunchStatus.isOk() || prepareLaunchStatus != ErrorStatus::NONE) { + return nullptr; + } + + preparedModelCallback->wait(); + ErrorStatus prepareReturnStatus = preparedModelCallback->getStatus(); + sp preparedModel = preparedModelCallback->getPreparedModel(); + if (prepareReturnStatus != ErrorStatus::NONE || preparedModel == nullptr) { + return nullptr; + } + + return preparedModel; +} + +// create device test +TEST_F(NeuralnetworksHidlTest, CreateDevice) {} + +// status test +TEST_F(NeuralnetworksHidlTest, StatusTest) { + Return status = device->getStatus(); + ASSERT_TRUE(status.isOk()); + EXPECT_EQ(DeviceStatus::AVAILABLE, static_cast(status)); +} + +// initialization +TEST_F(NeuralnetworksHidlTest, GetCapabilitiesTest) { + Return ret = + device->getCapabilities([](ErrorStatus status, const Capabilities& capabilities) { + EXPECT_EQ(ErrorStatus::NONE, status); + EXPECT_LT(0.0f, capabilities.float32Performance.execTime); + EXPECT_LT(0.0f, capabilities.float32Performance.powerUsage); + EXPECT_LT(0.0f, capabilities.quantized8Performance.execTime); + EXPECT_LT(0.0f, capabilities.quantized8Performance.powerUsage); + }); + EXPECT_TRUE(ret.isOk()); +} + +// supported operations positive test +TEST_F(NeuralnetworksHidlTest, SupportedOperationsPositiveTest) { + Model model = createValidTestModel_1_1(); + Return ret = device->getSupportedOperations_1_1( + model, [&](ErrorStatus status, const hidl_vec& supported) { + EXPECT_EQ(ErrorStatus::NONE, status); + EXPECT_EQ(model.operations.size(), supported.size()); + }); + EXPECT_TRUE(ret.isOk()); +} + +// supported operations negative test 1 +TEST_F(NeuralnetworksHidlTest, SupportedOperationsNegativeTest1) { + Model model = createInvalidTestModel1_1_1(); + Return ret = device->getSupportedOperations_1_1( + model, [&](ErrorStatus status, const hidl_vec& supported) { + EXPECT_EQ(ErrorStatus::INVALID_ARGUMENT, status); + (void)supported; + }); + EXPECT_TRUE(ret.isOk()); +} + +// supported operations negative test 2 +TEST_F(NeuralnetworksHidlTest, SupportedOperationsNegativeTest2) { + Model model = createInvalidTestModel2_1_1(); + Return ret = device->getSupportedOperations_1_1( + model, [&](ErrorStatus status, const hidl_vec& supported) { + EXPECT_EQ(ErrorStatus::INVALID_ARGUMENT, status); + (void)supported; + }); + EXPECT_TRUE(ret.isOk()); +} + +// prepare simple model positive test +TEST_F(NeuralnetworksHidlTest, SimplePrepareModelPositiveTest) { + Model model = createValidTestModel_1_1(); + sp preparedModelCallback = new PreparedModelCallback(); + ASSERT_NE(nullptr, preparedModelCallback.get()); + Return prepareLaunchStatus = + device->prepareModel_1_1(model, preparedModelCallback); + ASSERT_TRUE(prepareLaunchStatus.isOk()); + EXPECT_EQ(ErrorStatus::NONE, static_cast(prepareLaunchStatus)); + + preparedModelCallback->wait(); + ErrorStatus prepareReturnStatus = preparedModelCallback->getStatus(); + EXPECT_EQ(ErrorStatus::NONE, prepareReturnStatus); + sp preparedModel = preparedModelCallback->getPreparedModel(); + EXPECT_NE(nullptr, preparedModel.get()); +} + +// prepare simple model negative test 1 +TEST_F(NeuralnetworksHidlTest, SimplePrepareModelNegativeTest1) { + Model model = createInvalidTestModel1_1_1(); + sp preparedModelCallback = new PreparedModelCallback(); + ASSERT_NE(nullptr, preparedModelCallback.get()); + Return prepareLaunchStatus = + device->prepareModel_1_1(model, preparedModelCallback); + ASSERT_TRUE(prepareLaunchStatus.isOk()); + EXPECT_EQ(ErrorStatus::INVALID_ARGUMENT, static_cast(prepareLaunchStatus)); + + preparedModelCallback->wait(); + ErrorStatus prepareReturnStatus = preparedModelCallback->getStatus(); + EXPECT_EQ(ErrorStatus::INVALID_ARGUMENT, prepareReturnStatus); + sp preparedModel = preparedModelCallback->getPreparedModel(); + EXPECT_EQ(nullptr, preparedModel.get()); +} + +// prepare simple model negative test 2 +TEST_F(NeuralnetworksHidlTest, SimplePrepareModelNegativeTest2) { + Model model = createInvalidTestModel2_1_1(); + sp preparedModelCallback = new PreparedModelCallback(); + ASSERT_NE(nullptr, preparedModelCallback.get()); + Return prepareLaunchStatus = + device->prepareModel_1_1(model, preparedModelCallback); + ASSERT_TRUE(prepareLaunchStatus.isOk()); + EXPECT_EQ(ErrorStatus::INVALID_ARGUMENT, static_cast(prepareLaunchStatus)); + + preparedModelCallback->wait(); + ErrorStatus prepareReturnStatus = preparedModelCallback->getStatus(); + EXPECT_EQ(ErrorStatus::INVALID_ARGUMENT, prepareReturnStatus); + sp preparedModel = preparedModelCallback->getPreparedModel(); + EXPECT_EQ(nullptr, preparedModel.get()); +} + +// execute simple graph positive test +TEST_F(NeuralnetworksHidlTest, SimpleExecuteGraphPositiveTest) { + std::vector outputData = {-1.0f, -1.0f, -1.0f, -1.0f}; + std::vector expectedData = {6.0f, 8.0f, 10.0f, 12.0f}; + const uint32_t OUTPUT = 1; + + sp preparedModel = doPrepareModelShortcut(device); + ASSERT_NE(nullptr, preparedModel.get()); + Request request = createValidTestRequest(); + + auto postWork = [&] { + sp outputMemory = mapMemory(request.pools[OUTPUT]); + if (outputMemory == nullptr) { + return false; + } + float* outputPtr = reinterpret_cast(static_cast(outputMemory->getPointer())); + if (outputPtr == nullptr) { + return false; + } + outputMemory->read(); + std::copy(outputPtr, outputPtr + outputData.size(), outputData.begin()); + outputMemory->commit(); + return true; + }; + + sp executionCallback = new ExecutionCallback(); + ASSERT_NE(nullptr, executionCallback.get()); + executionCallback->on_finish(postWork); + Return executeLaunchStatus = preparedModel->execute(request, executionCallback); + ASSERT_TRUE(executeLaunchStatus.isOk()); + EXPECT_EQ(ErrorStatus::NONE, static_cast(executeLaunchStatus)); + + executionCallback->wait(); + ErrorStatus executionReturnStatus = executionCallback->getStatus(); + EXPECT_EQ(ErrorStatus::NONE, executionReturnStatus); + EXPECT_EQ(expectedData, outputData); +} + +// execute simple graph negative test 1 +TEST_F(NeuralnetworksHidlTest, SimpleExecuteGraphNegativeTest1) { + sp preparedModel = doPrepareModelShortcut(device); + ASSERT_NE(nullptr, preparedModel.get()); + Request request = createInvalidTestRequest1(); + + sp executionCallback = new ExecutionCallback(); + ASSERT_NE(nullptr, executionCallback.get()); + Return executeLaunchStatus = preparedModel->execute(request, executionCallback); + ASSERT_TRUE(executeLaunchStatus.isOk()); + EXPECT_EQ(ErrorStatus::INVALID_ARGUMENT, static_cast(executeLaunchStatus)); + + executionCallback->wait(); + ErrorStatus executionReturnStatus = executionCallback->getStatus(); + EXPECT_EQ(ErrorStatus::INVALID_ARGUMENT, executionReturnStatus); +} + +// execute simple graph negative test 2 +TEST_F(NeuralnetworksHidlTest, SimpleExecuteGraphNegativeTest2) { + sp preparedModel = doPrepareModelShortcut(device); + ASSERT_NE(nullptr, preparedModel.get()); + Request request = createInvalidTestRequest2(); + + sp executionCallback = new ExecutionCallback(); + ASSERT_NE(nullptr, executionCallback.get()); + Return executeLaunchStatus = preparedModel->execute(request, executionCallback); + ASSERT_TRUE(executeLaunchStatus.isOk()); + EXPECT_EQ(ErrorStatus::INVALID_ARGUMENT, static_cast(executeLaunchStatus)); + + executionCallback->wait(); + ErrorStatus executionReturnStatus = executionCallback->getStatus(); + EXPECT_EQ(ErrorStatus::INVALID_ARGUMENT, executionReturnStatus); +} + +} // namespace functional +} // namespace vts +} // namespace V1_1 +} // namespace neuralnetworks +} // namespace hardware +} // namespace android + +using android::hardware::neuralnetworks::V1_1::vts::functional::NeuralnetworksHidlEnvironment; + +int main(int argc, char** argv) { + ::testing::AddGlobalTestEnvironment(NeuralnetworksHidlEnvironment::getInstance()); + ::testing::InitGoogleTest(&argc, argv); + NeuralnetworksHidlEnvironment::getInstance()->init(&argc, argv); + + int status = RUN_ALL_TESTS(); + return status; +} diff --git a/neuralnetworks/1.1/vts/functional/VtsHalNeuralnetworksV1_1GeneratedTest.cpp b/neuralnetworks/1.1/vts/functional/VtsHalNeuralnetworksV1_1GeneratedTest.cpp new file mode 100644 index 0000000000..025d9feda3 --- /dev/null +++ b/neuralnetworks/1.1/vts/functional/VtsHalNeuralnetworksV1_1GeneratedTest.cpp @@ -0,0 +1,80 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#define LOG_TAG "neuralnetworks_hidl_hal_test" + +#include "VtsHalNeuralnetworksV1_1.h" + +#include "Callbacks.h" +#include "TestHarness.h" + +#include +#include +#include +#include +#include + +using ::android::hardware::neuralnetworks::V1_0::IPreparedModel; +using ::android::hardware::neuralnetworks::V1_0::Capabilities; +using ::android::hardware::neuralnetworks::V1_0::DeviceStatus; +using ::android::hardware::neuralnetworks::V1_0::ErrorStatus; +using ::android::hardware::neuralnetworks::V1_0::FusedActivationFunc; +using ::android::hardware::neuralnetworks::V1_0::Operand; +using ::android::hardware::neuralnetworks::V1_0::OperandLifeTime; +using ::android::hardware::neuralnetworks::V1_0::OperandType; +using ::android::hardware::neuralnetworks::V1_0::Request; +using ::android::hardware::neuralnetworks::V1_1::IDevice; +using ::android::hardware::neuralnetworks::V1_1::Model; +using ::android::hardware::neuralnetworks::V1_1::Operation; +using ::android::hardware::neuralnetworks::V1_1::OperationType; +using ::android::hardware::Return; +using ::android::hardware::Void; +using ::android::hardware::hidl_memory; +using ::android::hardware::hidl_string; +using ::android::hardware::hidl_vec; +using ::android::hidl::allocator::V1_0::IAllocator; +using ::android::hidl::memory::V1_0::IMemory; +using ::android::sp; + +namespace android { +namespace hardware { +namespace neuralnetworks { + +namespace generated_tests { +using ::generated_tests::MixedTypedExampleType; +extern void Execute(sp&, std::function, std::function, + const std::vector&); +} // namespace generated_tests + +namespace V1_1 { +namespace vts { +namespace functional { +using ::android::hardware::neuralnetworks::V1_0::implementation::ExecutionCallback; +using ::android::hardware::neuralnetworks::V1_0::implementation::PreparedModelCallback; + +// Mixed-typed examples +typedef generated_tests::MixedTypedExampleType MixedTypedExample; + +// in frameworks/ml/nn/runtime/tests/generated/ +#include "all_generated_V1_0_vts_tests.cpp" +#include "all_generated_V1_1_vts_tests.cpp" + +} // namespace functional +} // namespace vts +} // namespace V1_1 +} // namespace neuralnetworks +} // namespace hardware +} // namespace android From e0d9b167a61669246cecb33d0279ceaaaade466f Mon Sep 17 00:00:00 2001 From: Michael Butler Date: Tue, 13 Feb 2018 18:56:49 -0800 Subject: [PATCH 3/6] NNAPI narrow evaluation for P -- HAL We have determined that for Android P it is sufficient to have a mechanism for a developer to specify on a per-model basis that it is acceptable for FLOAT32 operations to be carried out as if they were FLOAT16 operations. This CL extends the model to include information on these precision requirements. Bug: 63911257 Test: mm Test: NeuralNetworksTest Test: VtsHalNeuralnetworksV1_0TargetTest Merged-In: I5331e69c05c20c588035897322a6c7dceca374a0 Change-Id: I5331e69c05c20c588035897322a6c7dceca374a0 (cherry picked from commit 7ffe78ba76c679724a7d3deed1963196fde0e3ce) --- neuralnetworks/1.1/types.hal | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/neuralnetworks/1.1/types.hal b/neuralnetworks/1.1/types.hal index 18863d3e89..fae5dd0caa 100644 --- a/neuralnetworks/1.1/types.hal +++ b/neuralnetworks/1.1/types.hal @@ -330,4 +330,12 @@ struct Model { * equals OperandLifeTime::CONSTANT_REFERENCE. */ vec pools; + + /** + * 'true' indicates TENSOR_FLOAT32 may be calculated with range and/or + * precision as low as that of the IEEE 754 16-bit floating-point format. + * 'false' indicates TENSOR_FLOAT32 must be calculated using at least the + * range and precision of the IEEE 754 32-bit floating-point format. + */ + bool relaxComputationFloat32toFloat16; }; From 7578e51fdaa42f87619b6028cb85fdc03a92deeb Mon Sep 17 00:00:00 2001 From: Miao Wang Date: Thu, 22 Feb 2018 11:48:59 -0800 Subject: [PATCH 4/6] Minor modification to allow relaxed precision test Bug: 63911257 Test: mm Test: VTS test pass on walleye Merged-In: I41d9ea782262a894e7dbd00b782803c40b31c430 Change-Id: I41d9ea782262a894e7dbd00b782803c40b31c430 (cherry picked from commit 842e1a03319361a984cf62e51cbf0486d5f15c46) --- .../1.0/vts/functional/GeneratedTestHarness.cpp | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/neuralnetworks/1.0/vts/functional/GeneratedTestHarness.cpp b/neuralnetworks/1.0/vts/functional/GeneratedTestHarness.cpp index 5fe8415147..f0ce9382b4 100644 --- a/neuralnetworks/1.0/vts/functional/GeneratedTestHarness.cpp +++ b/neuralnetworks/1.0/vts/functional/GeneratedTestHarness.cpp @@ -66,7 +66,8 @@ void copy_back(MixedTyped* dst, const std::vector& ra, char* sr // Top level driver for models and examples generated by test_generator.py // Test driver for those generated from ml/nn/runtime/test/spec void EvaluatePreparedModel(sp& preparedModel, std::function is_ignored, - const std::vector& examples) { + const std::vector& examples, + float fpRange = 1e-5f) { const uint32_t INPUT = 0; const uint32_t OUTPUT = 1; @@ -174,7 +175,7 @@ void EvaluatePreparedModel(sp& preparedModel, std::function& device, std::function create_ } ASSERT_NE(nullptr, preparedModel.get()); - EvaluatePreparedModel(preparedModel, is_ignored, examples); + // If in relaxed mode, set the error range to be 5ULP of FP16. + float fpRange = !model.relaxComputationFloat32toFloat16 ? 1e-5f : 5.0f * 0.0009765625f; + EvaluatePreparedModel(preparedModel, is_ignored, examples, fpRange); } } // namespace generated_tests From a8cd639dc0c6e84c27f9b32590564b5eeae6a490 Mon Sep 17 00:00:00 2001 From: David Gross Date: Tue, 20 Feb 2018 14:56:00 -0800 Subject: [PATCH 5/6] Add Capabilities::relaxedFloat32toFloat16Performance for 1.1. Test: VtsHalNeuralnetworksV1_0TargetTest --hal_service_instance=android.hardware.neuralnetworks@1.0::IDevice/sample-all Test: VtsHalNeuralnetworksV1_0TargetTest --hal_service_instance=android.hardware.neuralnetworks@1.0::IDevice/hvx Test: VtsHalNeuralnetworksV1_1TargetTest --hal_service_instance=android.hardware.neuralnetworks@1.1::IDevice/sample-all Test: NeuralNetworksTest_static Bug: 63911257 Merged-In: I040b68d84504180ba3039e4690d0cd43c2c2649f Change-Id: I040b68d84504180ba3039e4690d0cd43c2c2649f (cherry picked from commit 4b458caa4e2e9e0dd052ca9e1185afa8bbe7cc65) --- neuralnetworks/1.1/IDevice.hal | 14 +++++++++++ neuralnetworks/1.1/types.hal | 23 +++++++++++++++++++ .../VtsHalNeuralnetworksV1_1BasicTest.cpp | 6 +++-- 3 files changed, 41 insertions(+), 2 deletions(-) diff --git a/neuralnetworks/1.1/IDevice.hal b/neuralnetworks/1.1/IDevice.hal index 9d3fc312a6..ca225554d5 100644 --- a/neuralnetworks/1.1/IDevice.hal +++ b/neuralnetworks/1.1/IDevice.hal @@ -24,6 +24,20 @@ import @1.0::IPreparedModelCallback; * This interface represents a device driver. */ interface IDevice extends @1.0::IDevice { + /** + * Gets the capabilities of a driver. + * + * Note that @1.1::Capabilities provides performance information + * on relaxed calculations, whereas @1.0::Capabilities does not. + * + * @return status Error status of the call, must be: + * - NONE if successful + * - DEVICE_UNAVAILABLE if driver is offline or busy + * - GENERAL_FAILURE if there is an unspecified error + * @return capabilities Capabilities of the driver. + */ + getCapabilities_1_1() generates (ErrorStatus status, Capabilities capabilities); + /** * Gets the supported operations in a model. * diff --git a/neuralnetworks/1.1/types.hal b/neuralnetworks/1.1/types.hal index fae5dd0caa..1d470d636f 100644 --- a/neuralnetworks/1.1/types.hal +++ b/neuralnetworks/1.1/types.hal @@ -18,6 +18,7 @@ package android.hardware.neuralnetworks@1.1; import @1.0::Operand; import @1.0::OperationType; +import @1.0::PerformanceInfo; /** * Operation types. @@ -258,6 +259,28 @@ enum OperationType : @1.0::OperationType { TRANSPOSE = 37, }; +/** + * The capabilities of a driver. + */ +struct Capabilities { + /** + * Driver performance when operating on float32 data. + */ + PerformanceInfo float32Performance; + + /** + * Driver performance when operating on asymmetric 8-bit quantized data. + */ + PerformanceInfo quantized8Performance; + + /** + * Driver performance when operating on float32 data but performing + * calculations with range and/or precision as low as that of the IEEE + * 754 16-bit floating-point format. + */ + PerformanceInfo relaxedFloat32toFloat16Performance; +}; + /** * Describes one operation of the model's graph. */ diff --git a/neuralnetworks/1.1/vts/functional/VtsHalNeuralnetworksV1_1BasicTest.cpp b/neuralnetworks/1.1/vts/functional/VtsHalNeuralnetworksV1_1BasicTest.cpp index 51eff2a019..d2e99a7064 100644 --- a/neuralnetworks/1.1/vts/functional/VtsHalNeuralnetworksV1_1BasicTest.cpp +++ b/neuralnetworks/1.1/vts/functional/VtsHalNeuralnetworksV1_1BasicTest.cpp @@ -29,7 +29,6 @@ #include using ::android::hardware::neuralnetworks::V1_0::IPreparedModel; -using ::android::hardware::neuralnetworks::V1_0::Capabilities; using ::android::hardware::neuralnetworks::V1_0::DeviceStatus; using ::android::hardware::neuralnetworks::V1_0::ErrorStatus; using ::android::hardware::neuralnetworks::V1_0::FusedActivationFunc; @@ -37,6 +36,7 @@ using ::android::hardware::neuralnetworks::V1_0::Operand; using ::android::hardware::neuralnetworks::V1_0::OperandLifeTime; using ::android::hardware::neuralnetworks::V1_0::OperandType; using ::android::hardware::neuralnetworks::V1_0::Request; +using ::android::hardware::neuralnetworks::V1_1::Capabilities; using ::android::hardware::neuralnetworks::V1_1::IDevice; using ::android::hardware::neuralnetworks::V1_1::Model; using ::android::hardware::neuralnetworks::V1_1::Operation; @@ -95,12 +95,14 @@ TEST_F(NeuralnetworksHidlTest, StatusTest) { // initialization TEST_F(NeuralnetworksHidlTest, GetCapabilitiesTest) { Return ret = - device->getCapabilities([](ErrorStatus status, const Capabilities& capabilities) { + device->getCapabilities_1_1([](ErrorStatus status, const Capabilities& capabilities) { EXPECT_EQ(ErrorStatus::NONE, status); EXPECT_LT(0.0f, capabilities.float32Performance.execTime); EXPECT_LT(0.0f, capabilities.float32Performance.powerUsage); EXPECT_LT(0.0f, capabilities.quantized8Performance.execTime); EXPECT_LT(0.0f, capabilities.quantized8Performance.powerUsage); + EXPECT_LT(0.0f, capabilities.relaxedFloat32toFloat16Performance.execTime); + EXPECT_LT(0.0f, capabilities.relaxedFloat32toFloat16Performance.powerUsage); }); EXPECT_TRUE(ret.isOk()); } From 1ae02d63df3d69c82774e4a4ca945ab5cc50fe3a Mon Sep 17 00:00:00 2001 From: Michael Butler Date: Mon, 26 Feb 2018 15:24:46 -0800 Subject: [PATCH 6/6] Relax NeuralNetwork's VTS positive and negative base tests There are some NN VTS tests that assume a service is able to generate a model consisting only of a floating point add operation. However, some drivers do not support floating point operations. This CL relaxes the test requirements to allow a test to be skipped if the service does not support floating point add. Bug: 72764145 Test: mma Test: VtsHalNeuralnetworksV1_0TargetTest Merged-In: I6b0644432680fc2f8098b5187795dc2953df03f9 Change-Id: I6b0644432680fc2f8098b5187795dc2953df03f9 (cherry picked from commit 4d5bb1097a34495212c09473b477dc97acb99264) --- .../vts/functional/GeneratedTestHarness.cpp | 46 ++++------ .../VtsHalNeuralnetworksV1_0BasicTest.cpp | 88 ++++++++++++------- .../VtsHalNeuralnetworksV1_1BasicTest.cpp | 85 +++++++++++------- 3 files changed, 128 insertions(+), 91 deletions(-) diff --git a/neuralnetworks/1.0/vts/functional/GeneratedTestHarness.cpp b/neuralnetworks/1.0/vts/functional/GeneratedTestHarness.cpp index f0ce9382b4..8646a4cbb0 100644 --- a/neuralnetworks/1.0/vts/functional/GeneratedTestHarness.cpp +++ b/neuralnetworks/1.0/vts/functional/GeneratedTestHarness.cpp @@ -186,35 +186,29 @@ void Execute(sp& device, std::function create_ // see if service can handle model bool fullySupportsModel = false; - ErrorStatus supportedStatus; - sp preparedModelCallback = new PreparedModelCallback(); - ASSERT_NE(nullptr, preparedModelCallback.get()); - Return supportedCall = device->getSupportedOperations( - model, [&](ErrorStatus status, const hidl_vec& supported) { - supportedStatus = status; + model, [&fullySupportsModel](ErrorStatus status, const hidl_vec& supported) { + ASSERT_EQ(ErrorStatus::NONE, status); ASSERT_NE(0ul, supported.size()); fullySupportsModel = std::all_of(supported.begin(), supported.end(), [](bool valid) { return valid; }); }); ASSERT_TRUE(supportedCall.isOk()); - ASSERT_EQ(ErrorStatus::NONE, supportedStatus); + + // launch prepare model + sp preparedModelCallback = new PreparedModelCallback(); + ASSERT_NE(nullptr, preparedModelCallback.get()); Return prepareLaunchStatus = device->prepareModel(model, preparedModelCallback); ASSERT_TRUE(prepareLaunchStatus.isOk()); + ASSERT_EQ(ErrorStatus::NONE, static_cast(prepareLaunchStatus)); // retrieve prepared model preparedModelCallback->wait(); ErrorStatus prepareReturnStatus = preparedModelCallback->getStatus(); sp preparedModel = preparedModelCallback->getPreparedModel(); - if (fullySupportsModel) { - EXPECT_EQ(ErrorStatus::NONE, prepareReturnStatus); - } else { - EXPECT_TRUE(prepareReturnStatus == ErrorStatus::NONE || - prepareReturnStatus == ErrorStatus::GENERAL_FAILURE); - } // early termination if vendor service cannot fully prepare model - if (!fullySupportsModel && prepareReturnStatus == ErrorStatus::GENERAL_FAILURE) { + if (!fullySupportsModel && prepareReturnStatus != ErrorStatus::NONE) { ASSERT_EQ(nullptr, preparedModel.get()); LOG(INFO) << "NN VTS: Early termination of test because vendor service cannot " "prepare model that it does not support."; @@ -223,6 +217,7 @@ void Execute(sp& device, std::function create_ << std::endl; return; } + EXPECT_EQ(ErrorStatus::NONE, prepareReturnStatus); ASSERT_NE(nullptr, preparedModel.get()); EvaluatePreparedModel(preparedModel, is_ignored, examples); @@ -235,36 +230,30 @@ void Execute(sp& device, std::function create_ // see if service can handle model bool fullySupportsModel = false; - ErrorStatus supportedStatus; - sp preparedModelCallback = new PreparedModelCallback(); - ASSERT_NE(nullptr, preparedModelCallback.get()); - Return supportedCall = device->getSupportedOperations_1_1( - model, [&](ErrorStatus status, const hidl_vec& supported) { - supportedStatus = status; + model, [&fullySupportsModel](ErrorStatus status, const hidl_vec& supported) { + ASSERT_EQ(ErrorStatus::NONE, status); ASSERT_NE(0ul, supported.size()); fullySupportsModel = std::all_of(supported.begin(), supported.end(), [](bool valid) { return valid; }); }); ASSERT_TRUE(supportedCall.isOk()); - ASSERT_EQ(ErrorStatus::NONE, supportedStatus); + + // launch prepare model + sp preparedModelCallback = new PreparedModelCallback(); + ASSERT_NE(nullptr, preparedModelCallback.get()); Return prepareLaunchStatus = device->prepareModel_1_1(model, preparedModelCallback); ASSERT_TRUE(prepareLaunchStatus.isOk()); + ASSERT_EQ(ErrorStatus::NONE, static_cast(prepareLaunchStatus)); // retrieve prepared model preparedModelCallback->wait(); ErrorStatus prepareReturnStatus = preparedModelCallback->getStatus(); sp preparedModel = preparedModelCallback->getPreparedModel(); - if (fullySupportsModel) { - EXPECT_EQ(ErrorStatus::NONE, prepareReturnStatus); - } else { - EXPECT_TRUE(prepareReturnStatus == ErrorStatus::NONE || - prepareReturnStatus == ErrorStatus::GENERAL_FAILURE); - } // early termination if vendor service cannot fully prepare model - if (!fullySupportsModel && prepareReturnStatus == ErrorStatus::GENERAL_FAILURE) { + if (!fullySupportsModel && prepareReturnStatus != ErrorStatus::NONE) { ASSERT_EQ(nullptr, preparedModel.get()); LOG(INFO) << "NN VTS: Early termination of test because vendor service cannot " "prepare model that it does not support."; @@ -273,6 +262,7 @@ void Execute(sp& device, std::function create_ << std::endl; return; } + EXPECT_EQ(ErrorStatus::NONE, prepareReturnStatus); ASSERT_NE(nullptr, preparedModel.get()); // If in relaxed mode, set the error range to be 5ULP of FP16. diff --git a/neuralnetworks/1.0/vts/functional/VtsHalNeuralnetworksV1_0BasicTest.cpp b/neuralnetworks/1.0/vts/functional/VtsHalNeuralnetworksV1_0BasicTest.cpp index e838997a8d..59e5b80612 100644 --- a/neuralnetworks/1.0/vts/functional/VtsHalNeuralnetworksV1_0BasicTest.cpp +++ b/neuralnetworks/1.0/vts/functional/VtsHalNeuralnetworksV1_0BasicTest.cpp @@ -52,26 +52,51 @@ namespace functional { using ::android::hardware::neuralnetworks::V1_0::implementation::ExecutionCallback; using ::android::hardware::neuralnetworks::V1_0::implementation::PreparedModelCallback; -inline sp doPrepareModelShortcut(sp& device) { +static void doPrepareModelShortcut(const sp& device, sp* preparedModel) { + ASSERT_NE(nullptr, preparedModel); Model model = createValidTestModel_1_0(); - sp preparedModelCallback = new PreparedModelCallback(); - if (preparedModelCallback == nullptr) { - return nullptr; - } - Return prepareLaunchStatus = device->prepareModel(model, preparedModelCallback); - if (!prepareLaunchStatus.isOk() || prepareLaunchStatus != ErrorStatus::NONE) { - return nullptr; - } + // see if service can handle model + bool fullySupportsModel = false; + Return supportedOpsLaunchStatus = device->getSupportedOperations( + model, [&fullySupportsModel](ErrorStatus status, const hidl_vec& supported) { + ASSERT_EQ(ErrorStatus::NONE, status); + ASSERT_NE(0ul, supported.size()); + fullySupportsModel = + std::all_of(supported.begin(), supported.end(), [](bool valid) { return valid; }); + }); + ASSERT_TRUE(supportedOpsLaunchStatus.isOk()); + // launch prepare model + sp preparedModelCallback = new PreparedModelCallback(); + ASSERT_NE(nullptr, preparedModelCallback.get()); + Return prepareLaunchStatus = device->prepareModel(model, preparedModelCallback); + ASSERT_TRUE(prepareLaunchStatus.isOk()); + ASSERT_EQ(ErrorStatus::NONE, static_cast(prepareLaunchStatus)); + + // retrieve prepared model preparedModelCallback->wait(); ErrorStatus prepareReturnStatus = preparedModelCallback->getStatus(); - sp preparedModel = preparedModelCallback->getPreparedModel(); - if (prepareReturnStatus != ErrorStatus::NONE || preparedModel == nullptr) { - return nullptr; - } + *preparedModel = preparedModelCallback->getPreparedModel(); - return preparedModel; + // The getSupportedOperations call returns a list of operations that are + // guaranteed not to fail if prepareModel is called, and + // 'fullySupportsModel' is true i.f.f. the entire model is guaranteed. + // If a driver has any doubt that it can prepare an operation, it must + // return false. So here, if a driver isn't sure if it can support an + // operation, but reports that it successfully prepared the model, the test + // can continue. + if (!fullySupportsModel && prepareReturnStatus != ErrorStatus::NONE) { + ASSERT_EQ(nullptr, preparedModel->get()); + LOG(INFO) << "NN VTS: Early termination of test because vendor service cannot " + "prepare model that it does not support."; + std::cout << "[ ] Early termination of test because vendor service cannot " + "prepare model that it does not support." + << std::endl; + return; + } + ASSERT_EQ(ErrorStatus::NONE, prepareReturnStatus); + ASSERT_NE(nullptr, preparedModel->get()); } // create device test @@ -132,18 +157,8 @@ TEST_F(NeuralnetworksHidlTest, SupportedOperationsNegativeTest2) { // prepare simple model positive test TEST_F(NeuralnetworksHidlTest, SimplePrepareModelPositiveTest) { - Model model = createValidTestModel_1_0(); - sp preparedModelCallback = new PreparedModelCallback(); - ASSERT_NE(nullptr, preparedModelCallback.get()); - Return prepareLaunchStatus = device->prepareModel(model, preparedModelCallback); - ASSERT_TRUE(prepareLaunchStatus.isOk()); - EXPECT_EQ(ErrorStatus::NONE, static_cast(prepareLaunchStatus)); - - preparedModelCallback->wait(); - ErrorStatus prepareReturnStatus = preparedModelCallback->getStatus(); - EXPECT_EQ(ErrorStatus::NONE, prepareReturnStatus); - sp preparedModel = preparedModelCallback->getPreparedModel(); - EXPECT_NE(nullptr, preparedModel.get()); + sp preparedModel; + doPrepareModelShortcut(device, &preparedModel); } // prepare simple model negative test 1 @@ -184,8 +199,11 @@ TEST_F(NeuralnetworksHidlTest, SimpleExecuteGraphPositiveTest) { std::vector expectedData = {6.0f, 8.0f, 10.0f, 12.0f}; const uint32_t OUTPUT = 1; - sp preparedModel = doPrepareModelShortcut(device); - ASSERT_NE(nullptr, preparedModel.get()); + sp preparedModel; + ASSERT_NO_FATAL_FAILURE(doPrepareModelShortcut(device, &preparedModel)); + if (preparedModel == nullptr) { + return; + } Request request = createValidTestRequest(); auto postWork = [&] { @@ -218,8 +236,11 @@ TEST_F(NeuralnetworksHidlTest, SimpleExecuteGraphPositiveTest) { // execute simple graph negative test 1 TEST_F(NeuralnetworksHidlTest, SimpleExecuteGraphNegativeTest1) { - sp preparedModel = doPrepareModelShortcut(device); - ASSERT_NE(nullptr, preparedModel.get()); + sp preparedModel; + ASSERT_NO_FATAL_FAILURE(doPrepareModelShortcut(device, &preparedModel)); + if (preparedModel == nullptr) { + return; + } Request request = createInvalidTestRequest1(); sp executionCallback = new ExecutionCallback(); @@ -235,8 +256,11 @@ TEST_F(NeuralnetworksHidlTest, SimpleExecuteGraphNegativeTest1) { // execute simple graph negative test 2 TEST_F(NeuralnetworksHidlTest, SimpleExecuteGraphNegativeTest2) { - sp preparedModel = doPrepareModelShortcut(device); - ASSERT_NE(nullptr, preparedModel.get()); + sp preparedModel; + ASSERT_NO_FATAL_FAILURE(doPrepareModelShortcut(device, &preparedModel)); + if (preparedModel == nullptr) { + return; + } Request request = createInvalidTestRequest2(); sp executionCallback = new ExecutionCallback(); diff --git a/neuralnetworks/1.1/vts/functional/VtsHalNeuralnetworksV1_1BasicTest.cpp b/neuralnetworks/1.1/vts/functional/VtsHalNeuralnetworksV1_1BasicTest.cpp index d2e99a7064..17f6744c2b 100644 --- a/neuralnetworks/1.1/vts/functional/VtsHalNeuralnetworksV1_1BasicTest.cpp +++ b/neuralnetworks/1.1/vts/functional/VtsHalNeuralnetworksV1_1BasicTest.cpp @@ -59,27 +59,52 @@ namespace functional { using ::android::hardware::neuralnetworks::V1_0::implementation::ExecutionCallback; using ::android::hardware::neuralnetworks::V1_0::implementation::PreparedModelCallback; -inline sp doPrepareModelShortcut(sp& device) { +static void doPrepareModelShortcut(const sp& device, sp* preparedModel) { + ASSERT_NE(nullptr, preparedModel); Model model = createValidTestModel_1_1(); + // see if service can handle model + bool fullySupportsModel = false; + Return supportedOpsLaunchStatus = device->getSupportedOperations_1_1( + model, [&fullySupportsModel](ErrorStatus status, const hidl_vec& supported) { + ASSERT_EQ(ErrorStatus::NONE, status); + ASSERT_NE(0ul, supported.size()); + fullySupportsModel = + std::all_of(supported.begin(), supported.end(), [](bool valid) { return valid; }); + }); + ASSERT_TRUE(supportedOpsLaunchStatus.isOk()); + + // launch prepare model sp preparedModelCallback = new PreparedModelCallback(); - if (preparedModelCallback == nullptr) { - return nullptr; - } + ASSERT_NE(nullptr, preparedModelCallback.get()); Return prepareLaunchStatus = device->prepareModel_1_1(model, preparedModelCallback); - if (!prepareLaunchStatus.isOk() || prepareLaunchStatus != ErrorStatus::NONE) { - return nullptr; - } + ASSERT_TRUE(prepareLaunchStatus.isOk()); + ASSERT_EQ(ErrorStatus::NONE, static_cast(prepareLaunchStatus)); + // retrieve prepared model preparedModelCallback->wait(); ErrorStatus prepareReturnStatus = preparedModelCallback->getStatus(); - sp preparedModel = preparedModelCallback->getPreparedModel(); - if (prepareReturnStatus != ErrorStatus::NONE || preparedModel == nullptr) { - return nullptr; - } + *preparedModel = preparedModelCallback->getPreparedModel(); - return preparedModel; + // The getSupportedOperations call returns a list of operations that are + // guaranteed not to fail if prepareModel is called, and + // 'fullySupportsModel' is true i.f.f. the entire model is guaranteed. + // If a driver has any doubt that it can prepare an operation, it must + // return false. So here, if a driver isn't sure if it can support an + // operation, but reports that it successfully prepared the model, the test + // can continue. + if (!fullySupportsModel && prepareReturnStatus != ErrorStatus::NONE) { + ASSERT_EQ(nullptr, preparedModel->get()); + LOG(INFO) << "NN VTS: Early termination of test because vendor service cannot " + "prepare model that it does not support."; + std::cout << "[ ] Early termination of test because vendor service cannot " + "prepare model that it does not support." + << std::endl; + return; + } + ASSERT_EQ(ErrorStatus::NONE, prepareReturnStatus); + ASSERT_NE(nullptr, preparedModel->get()); } // create device test @@ -142,19 +167,8 @@ TEST_F(NeuralnetworksHidlTest, SupportedOperationsNegativeTest2) { // prepare simple model positive test TEST_F(NeuralnetworksHidlTest, SimplePrepareModelPositiveTest) { - Model model = createValidTestModel_1_1(); - sp preparedModelCallback = new PreparedModelCallback(); - ASSERT_NE(nullptr, preparedModelCallback.get()); - Return prepareLaunchStatus = - device->prepareModel_1_1(model, preparedModelCallback); - ASSERT_TRUE(prepareLaunchStatus.isOk()); - EXPECT_EQ(ErrorStatus::NONE, static_cast(prepareLaunchStatus)); - - preparedModelCallback->wait(); - ErrorStatus prepareReturnStatus = preparedModelCallback->getStatus(); - EXPECT_EQ(ErrorStatus::NONE, prepareReturnStatus); - sp preparedModel = preparedModelCallback->getPreparedModel(); - EXPECT_NE(nullptr, preparedModel.get()); + sp preparedModel; + doPrepareModelShortcut(device, &preparedModel); } // prepare simple model negative test 1 @@ -197,8 +211,11 @@ TEST_F(NeuralnetworksHidlTest, SimpleExecuteGraphPositiveTest) { std::vector expectedData = {6.0f, 8.0f, 10.0f, 12.0f}; const uint32_t OUTPUT = 1; - sp preparedModel = doPrepareModelShortcut(device); - ASSERT_NE(nullptr, preparedModel.get()); + sp preparedModel; + ASSERT_NO_FATAL_FAILURE(doPrepareModelShortcut(device, &preparedModel)); + if (preparedModel == nullptr) { + return; + } Request request = createValidTestRequest(); auto postWork = [&] { @@ -231,8 +248,11 @@ TEST_F(NeuralnetworksHidlTest, SimpleExecuteGraphPositiveTest) { // execute simple graph negative test 1 TEST_F(NeuralnetworksHidlTest, SimpleExecuteGraphNegativeTest1) { - sp preparedModel = doPrepareModelShortcut(device); - ASSERT_NE(nullptr, preparedModel.get()); + sp preparedModel; + ASSERT_NO_FATAL_FAILURE(doPrepareModelShortcut(device, &preparedModel)); + if (preparedModel == nullptr) { + return; + } Request request = createInvalidTestRequest1(); sp executionCallback = new ExecutionCallback(); @@ -248,8 +268,11 @@ TEST_F(NeuralnetworksHidlTest, SimpleExecuteGraphNegativeTest1) { // execute simple graph negative test 2 TEST_F(NeuralnetworksHidlTest, SimpleExecuteGraphNegativeTest2) { - sp preparedModel = doPrepareModelShortcut(device); - ASSERT_NE(nullptr, preparedModel.get()); + sp preparedModel; + ASSERT_NO_FATAL_FAILURE(doPrepareModelShortcut(device, &preparedModel)); + if (preparedModel == nullptr) { + return; + } Request request = createInvalidTestRequest2(); sp executionCallback = new ExecutionCallback();