From a316581b217dcc35d8cafd902f650deb25b3d270 Mon Sep 17 00:00:00 2001 From: Xusong Wang Date: Mon, 19 Nov 2018 18:26:08 -0800 Subject: [PATCH] Test dynamic output shape in GeneratedTests. Enable VTS unit test for dynamic output shape deduction. Only test dynamic output shape for V1_2::IDevice with V1_2::Model. Bug: 73506513 Test: VtsHalNeuralnetworksV1_xTargetTest with 1.2 sample driver Change-Id: I4134e1ec54a15554eb8533134897279651b57da3 --- .../vts/functional/GeneratedTestHarness.cpp | 68 ++++++++++++------- neuralnetworks/1.2/vts/functional/Android.bp | 9 +++ .../1.2/vts/functional/GeneratedTests.cpp | 3 +- .../1.2/vts/functional/GeneratedTestsV1_0.cpp | 3 +- .../1.2/vts/functional/GeneratedTestsV1_1.cpp | 3 +- .../1.2/vts/functional/ValidateModel.cpp | 3 + .../1.2/vts/functional/VtsHalNeuralnetworks.h | 3 + 7 files changed, 65 insertions(+), 27 deletions(-) diff --git a/neuralnetworks/1.0/vts/functional/GeneratedTestHarness.cpp b/neuralnetworks/1.0/vts/functional/GeneratedTestHarness.cpp index b5a860771f..de0e494bf8 100644 --- a/neuralnetworks/1.0/vts/functional/GeneratedTestHarness.cpp +++ b/neuralnetworks/1.0/vts/functional/GeneratedTestHarness.cpp @@ -51,14 +51,13 @@ using ::test_helper::for_each; using ::test_helper::Int32Operands; using ::test_helper::MixedTyped; using ::test_helper::MixedTypedExample; -using ::test_helper::MixedTypedIndex; using ::test_helper::Quant8Operands; using ::test_helper::resize_accordingly; template -void copy_back_(MixedTyped* dst, const std::vector& ra, char* src) { - MixedTyped& test = *dst; - for_each(test, [&ra, src](int index, std::vector& m) { +void copy_back_(std::map>* dst, const std::vector& ra, + char* src) { + for_each(*dst, [&ra, src](int index, std::vector& m) { ASSERT_EQ(m.size(), ra[index].location.length / sizeof(T)); char* begin = src + ra[index].location.offset; memcpy(m.data(), begin, ra[index].location.length); @@ -66,14 +65,14 @@ void copy_back_(MixedTyped* dst, const std::vector& ra, char* s } void copy_back(MixedTyped* dst, const std::vector& ra, char* src) { - copy_back_(dst, ra, src); - copy_back_(dst, ra, src); - copy_back_(dst, ra, src); - copy_back_(dst, ra, src); - copy_back_<_Float16>(dst, ra, src); - copy_back_(dst, ra, src); - copy_back_(dst, ra, src); - static_assert(7 == std::tuple_size::value, + copy_back_(&dst->float32Operands, ra, src); + copy_back_(&dst->int32Operands, ra, src); + copy_back_(&dst->quant8Operands, ra, src); + copy_back_(&dst->quant16Operands, ra, src); + copy_back_(&dst->float16Operands, ra, src); + copy_back_(&dst->bool8Operands, ra, src); + copy_back_(&dst->quant8ChannelOperands, ra, src); + static_assert(7 == MixedTyped::kNumTypes, "Number of types in MixedTyped changed, but copy_back function wasn't updated"); } @@ -115,7 +114,8 @@ template void EvaluatePreparedModel(sp& preparedModel, std::function is_ignored, const std::vector& examples, bool hasRelaxedFloat32Model = false, float fpAtol = kDefaultAtol, - float fpRtol = kDefaultRtol, Synchronously sync = Synchronously::NO) { + float fpRtol = kDefaultRtol, Synchronously sync = Synchronously::NO, + bool testDynamicOutputShape = false) { const uint32_t INPUT = 0; const uint32_t OUTPUT = 1; @@ -125,7 +125,7 @@ void EvaluatePreparedModel(sp& preparedModel, std::function::index>(inputs).empty(); + const bool hasFloat16Inputs = !inputs.float16Operands.empty(); if (hasRelaxedFloat32Model || hasFloat16Inputs) { // TODO: Adjust the error limit based on testing. // If in relaxed mode, set the absolute tolerance to be 5ULP of FP16. @@ -237,10 +237,24 @@ void EvaluatePreparedModel(sp& preparedModel, std::function(executionReturnStatus); } + if (testDynamicOutputShape && executionStatus != ErrorStatus::NONE) { + LOG(INFO) << "NN VTS: Early termination of test because vendor service cannot " + "execute model that it does not support."; + std::cout << "[ ] Early termination of test because vendor service cannot " + "execute model that it does not support." + << std::endl; + return; + } ASSERT_EQ(ErrorStatus::NONE, executionStatus); - // TODO(xusongw): Check if the returned output shapes match with expectation once the - // sample driver implementation of dynamic output shape is finished. - ASSERT_EQ(outputShapes.size(), 0); + + // Go through all outputs, overwrite output dimensions with returned output shapes + if (testDynamicOutputShape) { + ASSERT_NE(outputShapes.size(), 0); + for_each(test.operandDimensions, + [&outputShapes](int idx, std::vector& dim) { + dim = outputShapes[idx].dimensions; + }); + } // validate results outputMemory->read(); @@ -261,9 +275,10 @@ void EvaluatePreparedModel(sp& preparedModel, std::function void EvaluatePreparedModel(sp& preparedModel, std::function is_ignored, const std::vector& examples, - bool hasRelaxedFloat32Model, Synchronously sync) { + bool hasRelaxedFloat32Model, Synchronously sync, + bool testDynamicOutputShape) { EvaluatePreparedModel(preparedModel, is_ignored, examples, hasRelaxedFloat32Model, kDefaultAtol, - kDefaultRtol, sync); + kDefaultRtol, sync, testDynamicOutputShape); } static void getPreparedModel(sp callback, @@ -319,7 +334,8 @@ void Execute(const sp& device, std::function c float fpAtol = 1e-5f, fpRtol = 5.0f * 1.1920928955078125e-7f; EvaluatePreparedModel(preparedModel, is_ignored, examples, - /*hasRelaxedFloat32Model=*/false, fpAtol, fpRtol); + /*hasRelaxedFloat32Model=*/false, fpAtol, fpRtol, Synchronously::NO, + /*testDynamicOutputShape=*/false); } void Execute(const sp& device, std::function create_model, @@ -365,12 +381,14 @@ void Execute(const sp& device, std::function c ASSERT_NE(nullptr, preparedModel.get()); EvaluatePreparedModel(preparedModel, is_ignored, examples, - model.relaxComputationFloat32toFloat16); + model.relaxComputationFloat32toFloat16, 1e-5f, 1e-5f, Synchronously::NO, + /*testDynamicOutputShape=*/false); } // TODO: Reduce code duplication. void Execute(const sp& device, std::function create_model, - std::function is_ignored, const std::vector& examples) { + std::function is_ignored, const std::vector& examples, + bool testDynamicOutputShape) { V1_2::Model model = create_model(); // see if service can handle model @@ -412,9 +430,11 @@ void Execute(const sp& device, std::function c ASSERT_NE(nullptr, preparedModel.get()); EvaluatePreparedModel(preparedModel, is_ignored, examples, - model.relaxComputationFloat32toFloat16, Synchronously::NO); + model.relaxComputationFloat32toFloat16, Synchronously::NO, + testDynamicOutputShape); EvaluatePreparedModel(preparedModel, is_ignored, examples, - model.relaxComputationFloat32toFloat16, Synchronously::YES); + model.relaxComputationFloat32toFloat16, Synchronously::YES, + testDynamicOutputShape); } } // namespace generated_tests diff --git a/neuralnetworks/1.2/vts/functional/Android.bp b/neuralnetworks/1.2/vts/functional/Android.bp index 5b119ee833..0cb9e16fcf 100644 --- a/neuralnetworks/1.2/vts/functional/Android.bp +++ b/neuralnetworks/1.2/vts/functional/Android.bp @@ -21,6 +21,9 @@ cc_test { srcs: [ "GeneratedTestsV1_0.cpp", ], + cflags: [ + "-DNN_TEST_DYNAMIC_OUTPUT_SHAPE" + ], test_suites: ["general-tests"], } @@ -31,6 +34,9 @@ cc_test { srcs: [ "GeneratedTestsV1_1.cpp", ], + cflags: [ + "-DNN_TEST_DYNAMIC_OUTPUT_SHAPE" + ], test_suites: ["general-tests"], } @@ -42,5 +48,8 @@ cc_test { "BasicTests.cpp", "GeneratedTests.cpp", ], + cflags: [ + "-DNN_TEST_DYNAMIC_OUTPUT_SHAPE" + ], test_suites: ["general-tests"], } diff --git a/neuralnetworks/1.2/vts/functional/GeneratedTests.cpp b/neuralnetworks/1.2/vts/functional/GeneratedTests.cpp index 9bff09cdb7..4bc891f3d1 100644 --- a/neuralnetworks/1.2/vts/functional/GeneratedTests.cpp +++ b/neuralnetworks/1.2/vts/functional/GeneratedTests.cpp @@ -33,7 +33,8 @@ namespace neuralnetworks { namespace generated_tests { using ::test_helper::MixedTypedExample; extern void Execute(const sp&, std::function, - std::function, const std::vector&); + std::function, const std::vector&, + bool testDynamicOutputShape = false); } // namespace generated_tests namespace V1_2 { diff --git a/neuralnetworks/1.2/vts/functional/GeneratedTestsV1_0.cpp b/neuralnetworks/1.2/vts/functional/GeneratedTestsV1_0.cpp index 56a61d4901..956926aaff 100644 --- a/neuralnetworks/1.2/vts/functional/GeneratedTestsV1_0.cpp +++ b/neuralnetworks/1.2/vts/functional/GeneratedTestsV1_0.cpp @@ -33,7 +33,8 @@ namespace neuralnetworks { namespace generated_tests { using ::test_helper::MixedTypedExample; extern void Execute(const sp&, std::function, - std::function, const std::vector&); + std::function, const std::vector&, + bool testDynamicOutputShape = false); } // namespace generated_tests namespace V1_2 { diff --git a/neuralnetworks/1.2/vts/functional/GeneratedTestsV1_1.cpp b/neuralnetworks/1.2/vts/functional/GeneratedTestsV1_1.cpp index 1c781eccf3..425690f321 100644 --- a/neuralnetworks/1.2/vts/functional/GeneratedTestsV1_1.cpp +++ b/neuralnetworks/1.2/vts/functional/GeneratedTestsV1_1.cpp @@ -33,7 +33,8 @@ namespace neuralnetworks { namespace generated_tests { using ::test_helper::MixedTypedExample; extern void Execute(const sp&, std::function, - std::function, const std::vector&); + std::function, const std::vector&, + bool testDynamicOutputShape = false); } // namespace generated_tests namespace V1_2 { diff --git a/neuralnetworks/1.2/vts/functional/ValidateModel.cpp b/neuralnetworks/1.2/vts/functional/ValidateModel.cpp index b1a0e53d46..1f9c99d891 100644 --- a/neuralnetworks/1.2/vts/functional/ValidateModel.cpp +++ b/neuralnetworks/1.2/vts/functional/ValidateModel.cpp @@ -172,6 +172,9 @@ static uint32_t getInvalidRank(OperandType type) { static void mutateOperandRankTest(const sp& device, const Model& model) { for (size_t operand = 0; operand < model.operands.size(); ++operand) { const uint32_t invalidRank = getInvalidRank(model.operands[operand].type); + if (invalidRank == 0) { + continue; + } const std::string message = "mutateOperandRankTest: operand " + std::to_string(operand) + " has rank of " + std::to_string(invalidRank); validate(device, message, model, [operand, invalidRank](Model* model) { diff --git a/neuralnetworks/1.2/vts/functional/VtsHalNeuralnetworks.h b/neuralnetworks/1.2/vts/functional/VtsHalNeuralnetworks.h index dedab8d996..c0c21bddf0 100644 --- a/neuralnetworks/1.2/vts/functional/VtsHalNeuralnetworks.h +++ b/neuralnetworks/1.2/vts/functional/VtsHalNeuralnetworks.h @@ -79,6 +79,9 @@ class ValidationTest : public NeuralnetworksHidlTest { // Tag for the generated tests class GeneratedTest : public NeuralnetworksHidlTest {}; +// Tag for the dynamic output shape tests +class DynamicOutputShapeTest : public NeuralnetworksHidlTest {}; + // Utility function to get PreparedModel from callback and downcast to V1_2. sp getPreparedModel_1_2( const sp& callback);