From fc2b10ec86ddda810250f130f9058cdf6656073a Mon Sep 17 00:00:00 2001 From: "Michael K. Sanders" Date: Wed, 7 Nov 2018 15:53:28 +0000 Subject: [PATCH 1/6] Separates VTS tests by HAL version. This prevents a fatal relocation error trying to link VtsHalNeuralnetworksV1_2TargetTest if it exceeds 1GB. Test: VtsHalNeuralnetworksV1_2TargetTest --hal_service_instance=android.hardware.neuralnetworks@1.2::IDevice/sample-all Test: VtsHalNeuralnetworksV1_2CompatV1_1TargetTest --hal_service_instance=android.hardware.neuralnetworks@1.2::IDevice/sample-all Test: VtsHalNeuralnetworksV1_2CompatV1_0TargetTest --hal_service_instance=android.hardware.neuralnetworks@1.2::IDevice/sample-all Test: VtsHalNeuralnetworksV1_1TargetTest --hal_service_instance=android.hardware.neuralnetworks@1.1::IDevice/sample-all Test: VtsHalNeuralnetworksV1_1CompatV1_0TargetTest --hal_service_instance=android.hardware.neuralnetworks@1.1::IDevice/sample-all Test: VtsHalNeuralnetworksV1_0TargetTest --hal_service_instance=android.hardware.neuralnetworks@1.0::IDevice/sample-all Bug: 119135172 Change-Id: Id5cfdeddc2f0d810b4eb47084640ae1dbf297ea1 Merged-In: Id5cfdeddc2f0d810b4eb47084640ae1dbf297ea1 (cherry-pick from 0fb3892c9359a8ade58cd2bcd3cf99096b6e1214) --- neuralnetworks/1.0/vts/functional/Android.bp | 28 +- neuralnetworks/1.1/vts/functional/Android.bp | 41 +-- .../1.1/vts/functional/GeneratedTests.cpp | 1 - .../1.1/vts/functional/GeneratedTestsV1_0.cpp | 56 +++ neuralnetworks/1.1/vts/functional/Models.h | 146 -------- .../1.1/vts/functional/ModelsV1_0.h | 201 +++++++++++ .../vts/functional/ValidationTestsV1_0.cpp | 50 +++ neuralnetworks/1.2/vts/functional/Android.bp | 51 ++- .../1.2/vts/functional/GeneratedTests.cpp | 2 - .../1.2/vts/functional/GeneratedTestsV1_0.cpp | 56 +++ .../1.2/vts/functional/GeneratedTestsV1_1.cpp | 56 +++ neuralnetworks/1.2/vts/functional/Models.h | 325 +----------------- .../1.2/vts/functional/ModelsV1_0.h | 202 +++++++++++ .../1.2/vts/functional/ModelsV1_1.h | 230 +++++++++++++ .../vts/functional/ValidationTestsV1_0.cpp | 50 +++ .../vts/functional/ValidationTestsV1_1.cpp | 50 +++ 16 files changed, 1008 insertions(+), 537 deletions(-) create mode 100644 neuralnetworks/1.1/vts/functional/GeneratedTestsV1_0.cpp create mode 100644 neuralnetworks/1.1/vts/functional/ModelsV1_0.h create mode 100644 neuralnetworks/1.1/vts/functional/ValidationTestsV1_0.cpp create mode 100644 neuralnetworks/1.2/vts/functional/GeneratedTestsV1_0.cpp create mode 100644 neuralnetworks/1.2/vts/functional/GeneratedTestsV1_1.cpp create mode 100644 neuralnetworks/1.2/vts/functional/ModelsV1_0.h create mode 100644 neuralnetworks/1.2/vts/functional/ModelsV1_1.h create mode 100644 neuralnetworks/1.2/vts/functional/ValidationTestsV1_0.cpp create mode 100644 neuralnetworks/1.2/vts/functional/ValidationTestsV1_1.cpp diff --git a/neuralnetworks/1.0/vts/functional/Android.bp b/neuralnetworks/1.0/vts/functional/Android.bp index ffba45c947..1b5e338bed 100644 --- a/neuralnetworks/1.0/vts/functional/Android.bp +++ b/neuralnetworks/1.0/vts/functional/Android.bp @@ -39,17 +39,14 @@ cc_library_static { ], } -cc_test { - name: "VtsHalNeuralnetworksV1_0TargetTest", +cc_defaults { + name: "VtsHalNeuralNetworksTargetTestDefaults", + defaults: ["VtsHalTargetTestDefaults"], srcs: [ - "BasicTests.cpp", - "GeneratedTests.cpp", "ValidateModel.cpp", "ValidateRequest.cpp", - "ValidationTests.cpp", "VtsHalNeuralnetworks.cpp", ], - defaults: ["VtsHalTargetTestDefaults"], static_libs: [ "android.hardware.neuralnetworks@1.0", "android.hardware.neuralnetworks@1.1", @@ -66,4 +63,23 @@ cc_test { "libneuralnetworks_generated_test_harness_headers", "libneuralnetworks_generated_tests", ], + // Bug: http://b/74200014 - Disable arm32 asan since it triggers internal + // error in ld.gold. + arch: { + arm: { + sanitize: { + never: true, + }, + }, + }, +} + +cc_test { + name: "VtsHalNeuralnetworksV1_0TargetTest", + defaults: ["VtsHalNeuralNetworksTargetTestDefaults"], + srcs: [ + "BasicTests.cpp", + "GeneratedTests.cpp", + "ValidationTests.cpp", + ], } diff --git a/neuralnetworks/1.1/vts/functional/Android.bp b/neuralnetworks/1.1/vts/functional/Android.bp index a1c0f1fba8..df1ac67927 100644 --- a/neuralnetworks/1.1/vts/functional/Android.bp +++ b/neuralnetworks/1.1/vts/functional/Android.bp @@ -14,40 +14,23 @@ // limitations under the License. // +// Tests for V1_0 models using the V1_1 HAL. +cc_test { + name: "VtsHalNeuralnetworksV1_1CompatV1_0TargetTest", + defaults: ["VtsHalNeuralNetworksTargetTestDefaults"], + srcs: [ + "GeneratedTestsV1_0.cpp", + "ValidationTestsV1_0.cpp", + ], +} + +// Tests for V1_1 models. cc_test { name: "VtsHalNeuralnetworksV1_1TargetTest", + defaults: ["VtsHalNeuralNetworksTargetTestDefaults"], srcs: [ "BasicTests.cpp", "GeneratedTests.cpp", - "ValidateModel.cpp", - "ValidateRequest.cpp", "ValidationTests.cpp", - "VtsHalNeuralnetworks.cpp", ], - defaults: ["VtsHalTargetTestDefaults"], - static_libs: [ - "android.hardware.neuralnetworks@1.0", - "android.hardware.neuralnetworks@1.1", - "android.hardware.neuralnetworks@1.2", - "android.hidl.allocator@1.0", - "android.hidl.memory@1.0", - "libgmock", - "libhidlmemory", - "libneuralnetworks_utils", - "VtsHalNeuralnetworksTest_utils", - ], - header_libs: [ - "libneuralnetworks_headers", - "libneuralnetworks_generated_test_harness_headers", - "libneuralnetworks_generated_tests", - ], - // Bug: http://b/74200014 - Disable arm32 asan since it triggers internal - // error in ld.gold. - arch: { - arm: { - sanitize: { - never: true, - }, - }, - }, } diff --git a/neuralnetworks/1.1/vts/functional/GeneratedTests.cpp b/neuralnetworks/1.1/vts/functional/GeneratedTests.cpp index 1f49904b4e..d16f181d12 100644 --- a/neuralnetworks/1.1/vts/functional/GeneratedTests.cpp +++ b/neuralnetworks/1.1/vts/functional/GeneratedTests.cpp @@ -46,7 +46,6 @@ using ::android::nn::allocateSharedMemory; using ::test_helper::MixedTypedExample; // in frameworks/ml/nn/runtime/tests/generated/ -#include "all_generated_V1_0_vts_tests.cpp" #include "all_generated_V1_1_vts_tests.cpp" } // namespace functional diff --git a/neuralnetworks/1.1/vts/functional/GeneratedTestsV1_0.cpp b/neuralnetworks/1.1/vts/functional/GeneratedTestsV1_0.cpp new file mode 100644 index 0000000000..e2acd7ded4 --- /dev/null +++ b/neuralnetworks/1.1/vts/functional/GeneratedTestsV1_0.cpp @@ -0,0 +1,56 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#define LOG_TAG "neuralnetworks_hidl_hal_test" + +#include "VtsHalNeuralnetworks.h" + +#include "Callbacks.h" +#include "TestHarness.h" +#include "Utils.h" + +#include +#include +#include + +namespace android { +namespace hardware { +namespace neuralnetworks { + +namespace generated_tests { +using ::test_helper::MixedTypedExample; +extern void Execute(const sp&, std::function, + std::function, const std::vector&); +} // namespace generated_tests + +namespace V1_1 { +namespace vts { +namespace functional { + +using ::android::hardware::neuralnetworks::V1_0::implementation::ExecutionCallback; +using ::android::hardware::neuralnetworks::V1_0::implementation::PreparedModelCallback; +using ::android::nn::allocateSharedMemory; +using ::test_helper::MixedTypedExample; + +// in frameworks/ml/nn/runtime/tests/generated/ +#include "all_generated_V1_0_vts_tests.cpp" + +} // namespace functional +} // namespace vts +} // namespace V1_1 +} // namespace neuralnetworks +} // namespace hardware +} // namespace android diff --git a/neuralnetworks/1.1/vts/functional/Models.h b/neuralnetworks/1.1/vts/functional/Models.h index 62bc95e4e4..57da0102da 100644 --- a/neuralnetworks/1.1/vts/functional/Models.h +++ b/neuralnetworks/1.1/vts/functional/Models.h @@ -34,193 +34,86 @@ namespace functional { using MixedTypedExample = test_helper::MixedTypedExample; #define FOR_EACH_TEST_MODEL(FN) \ - FN(add) \ - FN(add_broadcast_quant8) \ - FN(add_quant8) \ FN(add_relaxed) \ - FN(avg_pool_float_1) \ FN(avg_pool_float_1_relaxed) \ - FN(avg_pool_float_2) \ FN(avg_pool_float_2_relaxed) \ - FN(avg_pool_float_3) \ FN(avg_pool_float_3_relaxed) \ - FN(avg_pool_float_4) \ FN(avg_pool_float_4_relaxed) \ - FN(avg_pool_float_5) \ FN(avg_pool_float_5_relaxed) \ - FN(avg_pool_quant8_1) \ - FN(avg_pool_quant8_2) \ - FN(avg_pool_quant8_3) \ - FN(avg_pool_quant8_4) \ - FN(avg_pool_quant8_5) \ FN(batch_to_space) \ FN(batch_to_space_float_1) \ FN(batch_to_space_float_1_relaxed) \ FN(batch_to_space_quant8_1) \ FN(batch_to_space_relaxed) \ - FN(concat_float_1) \ FN(concat_float_1_relaxed) \ - FN(concat_float_2) \ FN(concat_float_2_relaxed) \ - FN(concat_float_3) \ FN(concat_float_3_relaxed) \ - FN(concat_quant8_1) \ - FN(concat_quant8_2) \ - FN(concat_quant8_3) \ - FN(conv_1_h3_w2_SAME) \ FN(conv_1_h3_w2_SAME_relaxed) \ - FN(conv_1_h3_w2_VALID) \ FN(conv_1_h3_w2_VALID_relaxed) \ - FN(conv_3_h3_w2_SAME) \ FN(conv_3_h3_w2_SAME_relaxed) \ - FN(conv_3_h3_w2_VALID) \ FN(conv_3_h3_w2_VALID_relaxed) \ - FN(conv_float) \ - FN(conv_float_2) \ FN(conv_float_2_relaxed) \ - FN(conv_float_channels) \ FN(conv_float_channels_relaxed) \ - FN(conv_float_channels_weights_as_inputs) \ FN(conv_float_channels_weights_as_inputs_relaxed) \ - FN(conv_float_large) \ FN(conv_float_large_relaxed) \ - FN(conv_float_large_weights_as_inputs) \ FN(conv_float_large_weights_as_inputs_relaxed) \ FN(conv_float_relaxed) \ - FN(conv_float_weights_as_inputs) \ FN(conv_float_weights_as_inputs_relaxed) \ - FN(conv_quant8) \ - FN(conv_quant8_2) \ - FN(conv_quant8_channels) \ - FN(conv_quant8_channels_weights_as_inputs) \ - FN(conv_quant8_large) \ - FN(conv_quant8_large_weights_as_inputs) \ - FN(conv_quant8_overflow) \ - FN(conv_quant8_overflow_weights_as_inputs) \ - FN(conv_quant8_weights_as_inputs) \ - FN(depth_to_space_float_1) \ FN(depth_to_space_float_1_relaxed) \ - FN(depth_to_space_float_2) \ FN(depth_to_space_float_2_relaxed) \ - FN(depth_to_space_float_3) \ FN(depth_to_space_float_3_relaxed) \ - FN(depth_to_space_quant8_1) \ - FN(depth_to_space_quant8_2) \ - FN(depthwise_conv) \ - FN(depthwise_conv2d_float) \ - FN(depthwise_conv2d_float_2) \ FN(depthwise_conv2d_float_2_relaxed) \ - FN(depthwise_conv2d_float_large) \ - FN(depthwise_conv2d_float_large_2) \ FN(depthwise_conv2d_float_large_2_relaxed) \ - FN(depthwise_conv2d_float_large_2_weights_as_inputs) \ FN(depthwise_conv2d_float_large_2_weights_as_inputs_relaxed) \ FN(depthwise_conv2d_float_large_relaxed) \ - FN(depthwise_conv2d_float_large_weights_as_inputs) \ FN(depthwise_conv2d_float_large_weights_as_inputs_relaxed) \ FN(depthwise_conv2d_float_relaxed) \ - FN(depthwise_conv2d_float_weights_as_inputs) \ FN(depthwise_conv2d_float_weights_as_inputs_relaxed) \ - FN(depthwise_conv2d_quant8) \ - FN(depthwise_conv2d_quant8_2) \ - FN(depthwise_conv2d_quant8_large) \ - FN(depthwise_conv2d_quant8_large_weights_as_inputs) \ - FN(depthwise_conv2d_quant8_weights_as_inputs) \ FN(depthwise_conv_relaxed) \ - FN(dequantize) \ FN(dequantize_relaxed) \ FN(div) \ FN(div_broadcast_float) \ FN(div_broadcast_float_relaxed) \ FN(div_relaxed) \ - FN(embedding_lookup) \ FN(embedding_lookup_relaxed) \ - FN(floor) \ FN(floor_relaxed) \ - FN(fully_connected_float) \ - FN(fully_connected_float_2) \ FN(fully_connected_float_2_relaxed) \ FN(fully_connected_float_4d_simple) \ FN(fully_connected_float_4d_simple_relaxed) \ - FN(fully_connected_float_large) \ FN(fully_connected_float_large_relaxed) \ - FN(fully_connected_float_large_weights_as_inputs) \ FN(fully_connected_float_large_weights_as_inputs_relaxed) \ FN(fully_connected_float_relaxed) \ - FN(fully_connected_float_weights_as_inputs) \ FN(fully_connected_float_weights_as_inputs_relaxed) \ - FN(fully_connected_quant8) \ - FN(fully_connected_quant8_2) \ - FN(fully_connected_quant8_large) \ - FN(fully_connected_quant8_large_weights_as_inputs) \ - FN(fully_connected_quant8_weights_as_inputs) \ - FN(hashtable_lookup_float) \ FN(hashtable_lookup_float_relaxed) \ - FN(hashtable_lookup_quant8) \ - FN(l2_normalization) \ - FN(l2_normalization_2) \ FN(l2_normalization_2_relaxed) \ - FN(l2_normalization_large) \ FN(l2_normalization_large_relaxed) \ FN(l2_normalization_relaxed) \ - FN(l2_pool_float) \ - FN(l2_pool_float_2) \ FN(l2_pool_float_2_relaxed) \ - FN(l2_pool_float_large) \ FN(l2_pool_float_large_relaxed) \ FN(l2_pool_float_relaxed) \ - FN(local_response_norm_float_1) \ FN(local_response_norm_float_1_relaxed) \ - FN(local_response_norm_float_2) \ FN(local_response_norm_float_2_relaxed) \ - FN(local_response_norm_float_3) \ FN(local_response_norm_float_3_relaxed) \ - FN(local_response_norm_float_4) \ FN(local_response_norm_float_4_relaxed) \ - FN(logistic_float_1) \ FN(logistic_float_1_relaxed) \ - FN(logistic_float_2) \ FN(logistic_float_2_relaxed) \ - FN(logistic_quant8_1) \ - FN(logistic_quant8_2) \ - FN(lsh_projection) \ - FN(lsh_projection_2) \ FN(lsh_projection_2_relaxed) \ FN(lsh_projection_relaxed) \ - FN(lsh_projection_weights_as_inputs) \ FN(lsh_projection_weights_as_inputs_relaxed) \ - FN(lstm) \ - FN(lstm2) \ FN(lstm2_relaxed) \ - FN(lstm2_state) \ - FN(lstm2_state2) \ FN(lstm2_state2_relaxed) \ FN(lstm2_state_relaxed) \ - FN(lstm3) \ FN(lstm3_relaxed) \ - FN(lstm3_state) \ - FN(lstm3_state2) \ FN(lstm3_state2_relaxed) \ - FN(lstm3_state3) \ FN(lstm3_state3_relaxed) \ FN(lstm3_state_relaxed) \ FN(lstm_relaxed) \ - FN(lstm_state) \ - FN(lstm_state2) \ FN(lstm_state2_relaxed) \ FN(lstm_state_relaxed) \ - FN(max_pool_float_1) \ FN(max_pool_float_1_relaxed) \ - FN(max_pool_float_2) \ FN(max_pool_float_2_relaxed) \ - FN(max_pool_float_3) \ FN(max_pool_float_3_relaxed) \ - FN(max_pool_float_4) \ FN(max_pool_float_4_relaxed) \ - FN(max_pool_quant8_1) \ - FN(max_pool_quant8_2) \ - FN(max_pool_quant8_3) \ - FN(max_pool_quant8_4) \ FN(mean) \ FN(mean_float_1) \ FN(mean_float_1_relaxed) \ @@ -229,57 +122,27 @@ using MixedTypedExample = test_helper::MixedTypedExample; FN(mean_quant8_1) \ FN(mean_quant8_2) \ FN(mean_relaxed) \ - FN(mobilenet_224_gender_basic_fixed) \ FN(mobilenet_224_gender_basic_fixed_relaxed) \ - FN(mobilenet_quantized) \ - FN(mul) \ - FN(mul_broadcast_quant8) \ - FN(mul_quant8) \ FN(mul_relaxed) \ - FN(mul_relu) \ FN(mul_relu_relaxed) \ FN(pad) \ FN(pad_float_1) \ FN(pad_float_1_relaxed) \ FN(pad_relaxed) \ - FN(relu1_float_1) \ FN(relu1_float_1_relaxed) \ - FN(relu1_float_2) \ FN(relu1_float_2_relaxed) \ - FN(relu1_quant8_1) \ - FN(relu1_quant8_2) \ - FN(relu6_float_1) \ FN(relu6_float_1_relaxed) \ - FN(relu6_float_2) \ FN(relu6_float_2_relaxed) \ - FN(relu6_quant8_1) \ - FN(relu6_quant8_2) \ - FN(relu_float_1) \ FN(relu_float_1_relaxed) \ - FN(relu_float_2) \ FN(relu_float_2_relaxed) \ - FN(relu_quant8_1) \ - FN(relu_quant8_2) \ - FN(reshape) \ - FN(reshape_quant8) \ - FN(reshape_quant8_weights_as_inputs) \ FN(reshape_relaxed) \ - FN(reshape_weights_as_inputs) \ FN(reshape_weights_as_inputs_relaxed) \ - FN(resize_bilinear) \ - FN(resize_bilinear_2) \ FN(resize_bilinear_2_relaxed) \ FN(resize_bilinear_relaxed) \ - FN(rnn) \ FN(rnn_relaxed) \ - FN(rnn_state) \ FN(rnn_state_relaxed) \ - FN(softmax_float_1) \ FN(softmax_float_1_relaxed) \ - FN(softmax_float_2) \ FN(softmax_float_2_relaxed) \ - FN(softmax_quant8_1) \ - FN(softmax_quant8_2) \ FN(space_to_batch) \ FN(space_to_batch_float_1) \ FN(space_to_batch_float_1_relaxed) \ @@ -291,14 +154,9 @@ using MixedTypedExample = test_helper::MixedTypedExample; FN(space_to_batch_quant8_2) \ FN(space_to_batch_quant8_3) \ FN(space_to_batch_relaxed) \ - FN(space_to_depth_float_1) \ FN(space_to_depth_float_1_relaxed) \ - FN(space_to_depth_float_2) \ FN(space_to_depth_float_2_relaxed) \ - FN(space_to_depth_float_3) \ FN(space_to_depth_float_3_relaxed) \ - FN(space_to_depth_quant8_1) \ - FN(space_to_depth_quant8_2) \ FN(squeeze) \ FN(squeeze_float_1) \ FN(squeeze_float_1_relaxed) \ @@ -343,13 +201,9 @@ using MixedTypedExample = test_helper::MixedTypedExample; FN(sub_broadcast_float) \ FN(sub_broadcast_float_relaxed) \ FN(sub_relaxed) \ - FN(svdf) \ - FN(svdf2) \ FN(svdf2_relaxed) \ FN(svdf_relaxed) \ - FN(svdf_state) \ FN(svdf_state_relaxed) \ - FN(tanh) \ FN(tanh_relaxed) \ FN(transpose) \ FN(transpose_float_1) \ diff --git a/neuralnetworks/1.1/vts/functional/ModelsV1_0.h b/neuralnetworks/1.1/vts/functional/ModelsV1_0.h new file mode 100644 index 0000000000..52c0346aaa --- /dev/null +++ b/neuralnetworks/1.1/vts/functional/ModelsV1_0.h @@ -0,0 +1,201 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef VTS_HAL_NEURALNETWORKS_V1_1_VTS_FUNCTIONAL_MODELS_V1_0_H +#define VTS_HAL_NEURALNETWORKS_V1_1_VTS_FUNCTIONAL_MODELS_V1_0_H + +#define LOG_TAG "neuralnetworks_hidl_hal_test" + +#include "TestHarness.h" + +#include +#include + +namespace android { +namespace hardware { +namespace neuralnetworks { +namespace V1_1 { +namespace vts { +namespace functional { + +using MixedTypedExample = test_helper::MixedTypedExample; + +#define FOR_EACH_TEST_MODEL(FN) \ + FN(add_broadcast_quant8) \ + FN(add) \ + FN(add_quant8) \ + FN(avg_pool_float_1) \ + FN(avg_pool_float_2) \ + FN(avg_pool_float_3) \ + FN(avg_pool_float_4) \ + FN(avg_pool_float_5) \ + FN(avg_pool_quant8_1) \ + FN(avg_pool_quant8_2) \ + FN(avg_pool_quant8_3) \ + FN(avg_pool_quant8_4) \ + FN(avg_pool_quant8_5) \ + FN(concat_float_1) \ + FN(concat_float_2) \ + FN(concat_float_3) \ + FN(concat_quant8_1) \ + FN(concat_quant8_2) \ + FN(concat_quant8_3) \ + FN(conv_1_h3_w2_SAME) \ + FN(conv_1_h3_w2_VALID) \ + FN(conv_3_h3_w2_SAME) \ + FN(conv_3_h3_w2_VALID) \ + FN(conv_float_2) \ + FN(conv_float_channels) \ + FN(conv_float_channels_weights_as_inputs) \ + FN(conv_float_large) \ + FN(conv_float_large_weights_as_inputs) \ + FN(conv_float) \ + FN(conv_float_weights_as_inputs) \ + FN(conv_quant8_2) \ + FN(conv_quant8_channels) \ + FN(conv_quant8_channels_weights_as_inputs) \ + FN(conv_quant8_large) \ + FN(conv_quant8_large_weights_as_inputs) \ + FN(conv_quant8) \ + FN(conv_quant8_overflow) \ + FN(conv_quant8_overflow_weights_as_inputs) \ + FN(conv_quant8_weights_as_inputs) \ + FN(depth_to_space_float_1) \ + FN(depth_to_space_float_2) \ + FN(depth_to_space_float_3) \ + FN(depth_to_space_quant8_1) \ + FN(depth_to_space_quant8_2) \ + FN(depthwise_conv2d_float_2) \ + FN(depthwise_conv2d_float_large_2) \ + FN(depthwise_conv2d_float_large_2_weights_as_inputs) \ + FN(depthwise_conv2d_float_large) \ + FN(depthwise_conv2d_float_large_weights_as_inputs) \ + FN(depthwise_conv2d_float) \ + FN(depthwise_conv2d_float_weights_as_inputs) \ + FN(depthwise_conv2d_quant8_2) \ + FN(depthwise_conv2d_quant8_large) \ + FN(depthwise_conv2d_quant8_large_weights_as_inputs) \ + FN(depthwise_conv2d_quant8) \ + FN(depthwise_conv2d_quant8_weights_as_inputs) \ + FN(depthwise_conv) \ + FN(dequantize) \ + FN(embedding_lookup) \ + FN(floor) \ + FN(fully_connected_float_2) \ + FN(fully_connected_float_large) \ + FN(fully_connected_float_large_weights_as_inputs) \ + FN(fully_connected_float) \ + FN(fully_connected_float_weights_as_inputs) \ + FN(fully_connected_quant8_2) \ + FN(fully_connected_quant8_large) \ + FN(fully_connected_quant8_large_weights_as_inputs) \ + FN(fully_connected_quant8) \ + FN(fully_connected_quant8_weights_as_inputs) \ + FN(hashtable_lookup_float) \ + FN(hashtable_lookup_quant8) \ + FN(l2_normalization_2) \ + FN(l2_normalization_large) \ + FN(l2_normalization) \ + FN(l2_pool_float_2) \ + FN(l2_pool_float_large) \ + FN(l2_pool_float) \ + FN(local_response_norm_float_1) \ + FN(local_response_norm_float_2) \ + FN(local_response_norm_float_3) \ + FN(local_response_norm_float_4) \ + FN(logistic_float_1) \ + FN(logistic_float_2) \ + FN(logistic_quant8_1) \ + FN(logistic_quant8_2) \ + FN(lsh_projection_2) \ + FN(lsh_projection) \ + FN(lsh_projection_weights_as_inputs) \ + FN(lstm2) \ + FN(lstm2_state2) \ + FN(lstm2_state) \ + FN(lstm3) \ + FN(lstm3_state2) \ + FN(lstm3_state3) \ + FN(lstm3_state) \ + FN(lstm) \ + FN(lstm_state2) \ + FN(lstm_state) \ + FN(max_pool_float_1) \ + FN(max_pool_float_2) \ + FN(max_pool_float_3) \ + FN(max_pool_float_4) \ + FN(max_pool_quant8_1) \ + FN(max_pool_quant8_2) \ + FN(max_pool_quant8_3) \ + FN(max_pool_quant8_4) \ + FN(mobilenet_224_gender_basic_fixed) \ + FN(mobilenet_quantized) \ + FN(mul_broadcast_quant8) \ + FN(mul) \ + FN(mul_quant8) \ + FN(mul_relu) \ + FN(relu1_float_1) \ + FN(relu1_float_2) \ + FN(relu1_quant8_1) \ + FN(relu1_quant8_2) \ + FN(relu6_float_1) \ + FN(relu6_float_2) \ + FN(relu6_quant8_1) \ + FN(relu6_quant8_2) \ + FN(relu_float_1) \ + FN(relu_float_2) \ + FN(relu_quant8_1) \ + FN(relu_quant8_2) \ + FN(reshape) \ + FN(reshape_quant8) \ + FN(reshape_quant8_weights_as_inputs) \ + FN(reshape_weights_as_inputs) \ + FN(resize_bilinear_2) \ + FN(resize_bilinear) \ + FN(rnn) \ + FN(rnn_state) \ + FN(softmax_float_1) \ + FN(softmax_float_2) \ + FN(softmax_quant8_1) \ + FN(softmax_quant8_2) \ + FN(space_to_depth_float_1) \ + FN(space_to_depth_float_2) \ + FN(space_to_depth_float_3) \ + FN(space_to_depth_quant8_1) \ + FN(space_to_depth_quant8_2) \ + FN(svdf2) \ + FN(svdf) \ + FN(svdf_state) \ + FN(tanh) + +#define FORWARD_DECLARE_GENERATED_OBJECTS(function) \ + namespace function { \ + extern std::vector examples; \ + Model createTestModel(); \ + } + +FOR_EACH_TEST_MODEL(FORWARD_DECLARE_GENERATED_OBJECTS) + +#undef FORWARD_DECLARE_GENERATED_OBJECTS + +} // namespace functional +} // namespace vts +} // namespace V1_1 +} // namespace neuralnetworks +} // namespace hardware +} // namespace android + +#endif // VTS_HAL_NEURALNETWORKS_V1_1_VTS_FUNCTIONAL_MODELS_V1_0_H diff --git a/neuralnetworks/1.1/vts/functional/ValidationTestsV1_0.cpp b/neuralnetworks/1.1/vts/functional/ValidationTestsV1_0.cpp new file mode 100644 index 0000000000..7e2af05501 --- /dev/null +++ b/neuralnetworks/1.1/vts/functional/ValidationTestsV1_0.cpp @@ -0,0 +1,50 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#define LOG_TAG "neuralnetworks_hidl_hal_test" + +#include "ModelsV1_0.h" +#include "VtsHalNeuralnetworks.h" + +namespace android { +namespace hardware { +namespace neuralnetworks { +namespace V1_1 { +namespace vts { +namespace functional { + +// forward declarations +std::vector createRequests(const std::vector& examples); + +// generate validation tests +#define VTS_CURRENT_TEST_CASE(TestName) \ + TEST_F(ValidationTest, TestName) { \ + const Model model = TestName::createTestModel(); \ + const std::vector requests = createRequests(TestName::examples); \ + validateModel(model); \ + validateRequests(model, requests); \ + } + +FOR_EACH_TEST_MODEL(VTS_CURRENT_TEST_CASE) + +#undef VTS_CURRENT_TEST_CASE + +} // namespace functional +} // namespace vts +} // namespace V1_1 +} // namespace neuralnetworks +} // namespace hardware +} // namespace android diff --git a/neuralnetworks/1.2/vts/functional/Android.bp b/neuralnetworks/1.2/vts/functional/Android.bp index 09d0dc3da6..087c12f811 100644 --- a/neuralnetworks/1.2/vts/functional/Android.bp +++ b/neuralnetworks/1.2/vts/functional/Android.bp @@ -14,40 +14,33 @@ // limitations under the License. // +// Tests for V1_0 models using the V1_2 HAL. +cc_test { + name: "VtsHalNeuralnetworksV1_2CompatV1_0TargetTest", + defaults: ["VtsHalNeuralNetworksTargetTestDefaults"], + srcs: [ + "GeneratedTestsV1_0.cpp", + "ValidationTestsV1_0.cpp", + ] +} + +// Tests for V1_1 models using the V1_2 HAL. +cc_test { + name: "VtsHalNeuralnetworksV1_2CompatV1_1TargetTest", + defaults: ["VtsHalNeuralNetworksTargetTestDefaults"], + srcs: [ + "GeneratedTestsV1_1.cpp", + "ValidationTestsV1_1.cpp", + ], +} + +// Tests for V1_2 models. cc_test { name: "VtsHalNeuralnetworksV1_2TargetTest", + defaults: ["VtsHalNeuralNetworksTargetTestDefaults"], srcs: [ "BasicTests.cpp", "GeneratedTests.cpp", - "ValidateModel.cpp", - "ValidateRequest.cpp", "ValidationTests.cpp", - "VtsHalNeuralnetworks.cpp", ], - defaults: ["VtsHalTargetTestDefaults"], - static_libs: [ - "android.hardware.neuralnetworks@1.0", - "android.hardware.neuralnetworks@1.1", - "android.hardware.neuralnetworks@1.2", - "android.hidl.allocator@1.0", - "android.hidl.memory@1.0", - "libgmock", - "libhidlmemory", - "libneuralnetworks_utils", - "VtsHalNeuralnetworksTest_utils", - ], - header_libs: [ - "libneuralnetworks_headers", - "libneuralnetworks_generated_test_harness_headers", - "libneuralnetworks_generated_tests", - ], - // Bug: http://b/74200014 - Disable arm32 asan since it triggers internal - // error in ld.gold. - arch: { - arm: { - sanitize: { - never: true, - }, - }, - }, } diff --git a/neuralnetworks/1.2/vts/functional/GeneratedTests.cpp b/neuralnetworks/1.2/vts/functional/GeneratedTests.cpp index e87fa6b6e1..0608139afc 100644 --- a/neuralnetworks/1.2/vts/functional/GeneratedTests.cpp +++ b/neuralnetworks/1.2/vts/functional/GeneratedTests.cpp @@ -46,8 +46,6 @@ using ::android::nn::allocateSharedMemory; using ::test_helper::MixedTypedExample; // in frameworks/ml/nn/runtime/tests/generated/ -#include "all_generated_V1_0_vts_tests.cpp" -#include "all_generated_V1_1_vts_tests.cpp" #include "all_generated_V1_2_vts_tests.cpp" } // namespace functional diff --git a/neuralnetworks/1.2/vts/functional/GeneratedTestsV1_0.cpp b/neuralnetworks/1.2/vts/functional/GeneratedTestsV1_0.cpp new file mode 100644 index 0000000000..8d685d190d --- /dev/null +++ b/neuralnetworks/1.2/vts/functional/GeneratedTestsV1_0.cpp @@ -0,0 +1,56 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#define LOG_TAG "neuralnetworks_hidl_hal_test" + +#include "VtsHalNeuralnetworks.h" + +#include "Callbacks.h" +#include "TestHarness.h" +#include "Utils.h" + +#include +#include +#include + +namespace android { +namespace hardware { +namespace neuralnetworks { + +namespace generated_tests { +using ::test_helper::MixedTypedExample; +extern void Execute(const sp&, std::function, + std::function, const std::vector&); +} // namespace generated_tests + +namespace V1_2 { +namespace vts { +namespace functional { + +using ::android::hardware::neuralnetworks::V1_0::implementation::ExecutionCallback; +using ::android::hardware::neuralnetworks::V1_0::implementation::PreparedModelCallback; +using ::android::nn::allocateSharedMemory; +using ::test_helper::MixedTypedExample; + +// in frameworks/ml/nn/runtime/tests/generated/ +#include "all_generated_V1_0_vts_tests.cpp" + +} // namespace functional +} // namespace vts +} // namespace V1_2 +} // namespace neuralnetworks +} // namespace hardware +} // namespace android diff --git a/neuralnetworks/1.2/vts/functional/GeneratedTestsV1_1.cpp b/neuralnetworks/1.2/vts/functional/GeneratedTestsV1_1.cpp new file mode 100644 index 0000000000..8dbb586b87 --- /dev/null +++ b/neuralnetworks/1.2/vts/functional/GeneratedTestsV1_1.cpp @@ -0,0 +1,56 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#define LOG_TAG "neuralnetworks_hidl_hal_test" + +#include "VtsHalNeuralnetworks.h" + +#include "Callbacks.h" +#include "TestHarness.h" +#include "Utils.h" + +#include +#include +#include + +namespace android { +namespace hardware { +namespace neuralnetworks { + +namespace generated_tests { +using ::test_helper::MixedTypedExample; +extern void Execute(const sp&, std::function, + std::function, const std::vector&); +} // namespace generated_tests + +namespace V1_2 { +namespace vts { +namespace functional { + +using ::android::hardware::neuralnetworks::V1_0::implementation::ExecutionCallback; +using ::android::hardware::neuralnetworks::V1_0::implementation::PreparedModelCallback; +using ::android::nn::allocateSharedMemory; +using ::test_helper::MixedTypedExample; + +// in frameworks/ml/nn/runtime/tests/generated/ +#include "all_generated_V1_1_vts_tests.cpp" + +} // namespace functional +} // namespace vts +} // namespace V1_2 +} // namespace neuralnetworks +} // namespace hardware +} // namespace android diff --git a/neuralnetworks/1.2/vts/functional/Models.h b/neuralnetworks/1.2/vts/functional/Models.h index 2d512feec2..2426ad0dba 100644 --- a/neuralnetworks/1.2/vts/functional/Models.h +++ b/neuralnetworks/1.2/vts/functional/Models.h @@ -34,330 +34,7 @@ namespace functional { using MixedTypedExample = test_helper::MixedTypedExample; -#define FOR_EACH_TEST_MODEL(FN) \ - FN(add) \ - FN(add_broadcast_quant8) \ - FN(add_quant8) \ - FN(add_relaxed) \ - FN(avg_pool_float_1) \ - FN(avg_pool_float_1_relaxed) \ - FN(avg_pool_float_2) \ - FN(avg_pool_float_2_relaxed) \ - FN(avg_pool_float_3) \ - FN(avg_pool_float_3_relaxed) \ - FN(avg_pool_float_4) \ - FN(avg_pool_float_4_relaxed) \ - FN(avg_pool_float_5) \ - FN(avg_pool_float_5_relaxed) \ - FN(avg_pool_quant8_1) \ - FN(avg_pool_quant8_2) \ - FN(avg_pool_quant8_3) \ - FN(avg_pool_quant8_4) \ - FN(avg_pool_quant8_5) \ - FN(batch_to_space) \ - FN(batch_to_space_float_1) \ - FN(batch_to_space_float_1_relaxed) \ - FN(batch_to_space_quant8_1) \ - FN(batch_to_space_relaxed) \ - FN(concat_float_1) \ - FN(concat_float_1_relaxed) \ - FN(concat_float_2) \ - FN(concat_float_2_relaxed) \ - FN(concat_float_3) \ - FN(concat_float_3_relaxed) \ - FN(concat_quant8_1) \ - FN(concat_quant8_2) \ - FN(concat_quant8_3) \ - FN(conv_1_h3_w2_SAME) \ - FN(conv_1_h3_w2_SAME_relaxed) \ - FN(conv_1_h3_w2_VALID) \ - FN(conv_1_h3_w2_VALID_relaxed) \ - FN(conv_3_h3_w2_SAME) \ - FN(conv_3_h3_w2_SAME_relaxed) \ - FN(conv_3_h3_w2_VALID) \ - FN(conv_3_h3_w2_VALID_relaxed) \ - FN(conv_float) \ - FN(conv_float_2) \ - FN(conv_float_2_relaxed) \ - FN(conv_float_channels) \ - FN(conv_float_channels_relaxed) \ - FN(conv_float_channels_weights_as_inputs) \ - FN(conv_float_channels_weights_as_inputs_relaxed) \ - FN(conv_float_large) \ - FN(conv_float_large_relaxed) \ - FN(conv_float_large_weights_as_inputs) \ - FN(conv_float_large_weights_as_inputs_relaxed) \ - FN(conv_float_relaxed) \ - FN(conv_float_weights_as_inputs) \ - FN(conv_float_weights_as_inputs_relaxed) \ - FN(conv_quant8) \ - FN(conv_quant8_2) \ - FN(conv_quant8_channels) \ - FN(conv_quant8_channels_weights_as_inputs) \ - FN(conv_quant8_large) \ - FN(conv_quant8_large_weights_as_inputs) \ - FN(conv_quant8_overflow) \ - FN(conv_quant8_overflow_weights_as_inputs) \ - FN(conv_quant8_weights_as_inputs) \ - FN(depth_to_space_float_1) \ - FN(depth_to_space_float_1_relaxed) \ - FN(depth_to_space_float_2) \ - FN(depth_to_space_float_2_relaxed) \ - FN(depth_to_space_float_3) \ - FN(depth_to_space_float_3_relaxed) \ - FN(depth_to_space_quant8_1) \ - FN(depth_to_space_quant8_2) \ - FN(depthwise_conv) \ - FN(depthwise_conv2d_float) \ - FN(depthwise_conv2d_float_2) \ - FN(depthwise_conv2d_float_2_relaxed) \ - FN(depthwise_conv2d_float_large) \ - FN(depthwise_conv2d_float_large_2) \ - FN(depthwise_conv2d_float_large_2_relaxed) \ - FN(depthwise_conv2d_float_large_2_weights_as_inputs) \ - FN(depthwise_conv2d_float_large_2_weights_as_inputs_relaxed) \ - FN(depthwise_conv2d_float_large_relaxed) \ - FN(depthwise_conv2d_float_large_weights_as_inputs) \ - FN(depthwise_conv2d_float_large_weights_as_inputs_relaxed) \ - FN(depthwise_conv2d_float_relaxed) \ - FN(depthwise_conv2d_float_weights_as_inputs) \ - FN(depthwise_conv2d_float_weights_as_inputs_relaxed) \ - FN(depthwise_conv2d_quant8) \ - FN(depthwise_conv2d_quant8_2) \ - FN(depthwise_conv2d_quant8_large) \ - FN(depthwise_conv2d_quant8_large_weights_as_inputs) \ - FN(depthwise_conv2d_quant8_weights_as_inputs) \ - FN(depthwise_conv_relaxed) \ - FN(dequantize) \ - FN(dequantize_relaxed) \ - FN(div) \ - FN(div_broadcast_float) \ - FN(div_broadcast_float_relaxed) \ - FN(div_relaxed) \ - FN(embedding_lookup) \ - FN(embedding_lookup_relaxed) \ - FN(floor) \ - FN(floor_relaxed) \ - FN(fully_connected_float) \ - FN(fully_connected_float_2) \ - FN(fully_connected_float_2_relaxed) \ - FN(fully_connected_float_4d_simple) \ - FN(fully_connected_float_4d_simple_relaxed) \ - FN(fully_connected_float_large) \ - FN(fully_connected_float_large_relaxed) \ - FN(fully_connected_float_large_weights_as_inputs) \ - FN(fully_connected_float_large_weights_as_inputs_relaxed) \ - FN(fully_connected_float_relaxed) \ - FN(fully_connected_float_weights_as_inputs) \ - FN(fully_connected_float_weights_as_inputs_relaxed) \ - FN(fully_connected_quant8) \ - FN(fully_connected_quant8_2) \ - FN(fully_connected_quant8_large) \ - FN(fully_connected_quant8_large_weights_as_inputs) \ - FN(fully_connected_quant8_weights_as_inputs) \ - FN(hashtable_lookup_float) \ - FN(hashtable_lookup_float_relaxed) \ - FN(hashtable_lookup_quant8) \ - FN(l2_normalization) \ - FN(l2_normalization_2) \ - FN(l2_normalization_2_relaxed) \ - FN(l2_normalization_large) \ - FN(l2_normalization_large_relaxed) \ - FN(l2_normalization_relaxed) \ - FN(l2_pool_float) \ - FN(l2_pool_float_2) \ - FN(l2_pool_float_2_relaxed) \ - FN(l2_pool_float_large) \ - FN(l2_pool_float_large_relaxed) \ - FN(l2_pool_float_relaxed) \ - FN(local_response_norm_float_1) \ - FN(local_response_norm_float_1_relaxed) \ - FN(local_response_norm_float_2) \ - FN(local_response_norm_float_2_relaxed) \ - FN(local_response_norm_float_3) \ - FN(local_response_norm_float_3_relaxed) \ - FN(local_response_norm_float_4) \ - FN(local_response_norm_float_4_relaxed) \ - FN(logistic_float_1) \ - FN(logistic_float_1_relaxed) \ - FN(logistic_float_2) \ - FN(logistic_float_2_relaxed) \ - FN(logistic_quant8_1) \ - FN(logistic_quant8_2) \ - FN(lsh_projection) \ - FN(lsh_projection_2) \ - FN(lsh_projection_2_relaxed) \ - FN(lsh_projection_relaxed) \ - FN(lsh_projection_weights_as_inputs) \ - FN(lsh_projection_weights_as_inputs_relaxed) \ - FN(lstm) \ - FN(lstm2) \ - FN(lstm2_relaxed) \ - FN(lstm2_state) \ - FN(lstm2_state2) \ - FN(lstm2_state2_relaxed) \ - FN(lstm2_state_relaxed) \ - FN(lstm3) \ - FN(lstm3_relaxed) \ - FN(lstm3_state) \ - FN(lstm3_state2) \ - FN(lstm3_state2_relaxed) \ - FN(lstm3_state3) \ - FN(lstm3_state3_relaxed) \ - FN(lstm3_state_relaxed) \ - FN(lstm_relaxed) \ - FN(lstm_state) \ - FN(lstm_state2) \ - FN(lstm_state2_relaxed) \ - FN(lstm_state_relaxed) \ - FN(max_pool_float_1) \ - FN(max_pool_float_1_relaxed) \ - FN(max_pool_float_2) \ - FN(max_pool_float_2_relaxed) \ - FN(max_pool_float_3) \ - FN(max_pool_float_3_relaxed) \ - FN(max_pool_float_4) \ - FN(max_pool_float_4_relaxed) \ - FN(max_pool_quant8_1) \ - FN(max_pool_quant8_2) \ - FN(max_pool_quant8_3) \ - FN(max_pool_quant8_4) \ - FN(mean) \ - FN(mean_float_1) \ - FN(mean_float_1_relaxed) \ - FN(mean_float_2) \ - FN(mean_float_2_relaxed) \ - FN(mean_quant8_1) \ - FN(mean_quant8_2) \ - FN(mean_relaxed) \ - FN(mobilenet_224_gender_basic_fixed) \ - FN(mobilenet_224_gender_basic_fixed_relaxed) \ - FN(mobilenet_quantized) \ - FN(mul) \ - FN(mul_broadcast_quant8) \ - FN(mul_quant8) \ - FN(mul_relaxed) \ - FN(mul_relu) \ - FN(mul_relu_relaxed) \ - FN(pad) \ - FN(pad_float_1) \ - FN(pad_float_1_relaxed) \ - FN(pad_relaxed) \ - FN(random_multinomial) \ - FN(relu1_float_1) \ - FN(relu1_float_1_relaxed) \ - FN(relu1_float_2) \ - FN(relu1_float_2_relaxed) \ - FN(relu1_quant8_1) \ - FN(relu1_quant8_2) \ - FN(relu6_float_1) \ - FN(relu6_float_1_relaxed) \ - FN(relu6_float_2) \ - FN(relu6_float_2_relaxed) \ - FN(relu6_quant8_1) \ - FN(relu6_quant8_2) \ - FN(relu_float_1) \ - FN(relu_float_1_relaxed) \ - FN(relu_float_2) \ - FN(relu_float_2_relaxed) \ - FN(relu_quant8_1) \ - FN(relu_quant8_2) \ - FN(reshape) \ - FN(reshape_quant8) \ - FN(reshape_quant8_weights_as_inputs) \ - FN(reshape_relaxed) \ - FN(reshape_weights_as_inputs) \ - FN(reshape_weights_as_inputs_relaxed) \ - FN(resize_bilinear) \ - FN(resize_bilinear_2) \ - FN(resize_bilinear_2_relaxed) \ - FN(resize_bilinear_relaxed) \ - FN(rnn) \ - FN(rnn_relaxed) \ - FN(rnn_state) \ - FN(rnn_state_relaxed) \ - FN(softmax_float_1) \ - FN(softmax_float_1_relaxed) \ - FN(softmax_float_2) \ - FN(softmax_float_2_relaxed) \ - FN(softmax_quant8_1) \ - FN(softmax_quant8_2) \ - FN(space_to_batch) \ - FN(space_to_batch_float_1) \ - FN(space_to_batch_float_1_relaxed) \ - FN(space_to_batch_float_2) \ - FN(space_to_batch_float_2_relaxed) \ - FN(space_to_batch_float_3) \ - FN(space_to_batch_float_3_relaxed) \ - FN(space_to_batch_quant8_1) \ - FN(space_to_batch_quant8_2) \ - FN(space_to_batch_quant8_3) \ - FN(space_to_batch_relaxed) \ - FN(space_to_depth_float_1) \ - FN(space_to_depth_float_1_relaxed) \ - FN(space_to_depth_float_2) \ - FN(space_to_depth_float_2_relaxed) \ - FN(space_to_depth_float_3) \ - FN(space_to_depth_float_3_relaxed) \ - FN(space_to_depth_quant8_1) \ - FN(space_to_depth_quant8_2) \ - FN(squeeze) \ - FN(squeeze_float_1) \ - FN(squeeze_float_1_relaxed) \ - FN(squeeze_quant8_1) \ - FN(squeeze_relaxed) \ - FN(strided_slice) \ - FN(strided_slice_float_1) \ - FN(strided_slice_float_10) \ - FN(strided_slice_float_10_relaxed) \ - FN(strided_slice_float_11) \ - FN(strided_slice_float_11_relaxed) \ - FN(strided_slice_float_1_relaxed) \ - FN(strided_slice_float_2) \ - FN(strided_slice_float_2_relaxed) \ - FN(strided_slice_float_3) \ - FN(strided_slice_float_3_relaxed) \ - FN(strided_slice_float_4) \ - FN(strided_slice_float_4_relaxed) \ - FN(strided_slice_float_5) \ - FN(strided_slice_float_5_relaxed) \ - FN(strided_slice_float_6) \ - FN(strided_slice_float_6_relaxed) \ - FN(strided_slice_float_7) \ - FN(strided_slice_float_7_relaxed) \ - FN(strided_slice_float_8) \ - FN(strided_slice_float_8_relaxed) \ - FN(strided_slice_float_9) \ - FN(strided_slice_float_9_relaxed) \ - FN(strided_slice_qaunt8_10) \ - FN(strided_slice_qaunt8_11) \ - FN(strided_slice_quant8_1) \ - FN(strided_slice_quant8_2) \ - FN(strided_slice_quant8_3) \ - FN(strided_slice_quant8_4) \ - FN(strided_slice_quant8_5) \ - FN(strided_slice_quant8_6) \ - FN(strided_slice_quant8_7) \ - FN(strided_slice_quant8_8) \ - FN(strided_slice_quant8_9) \ - FN(strided_slice_relaxed) \ - FN(sub) \ - FN(sub_broadcast_float) \ - FN(sub_broadcast_float_relaxed) \ - FN(sub_relaxed) \ - FN(svdf) \ - FN(svdf2) \ - FN(svdf2_relaxed) \ - FN(svdf_relaxed) \ - FN(svdf_state) \ - FN(svdf_state_relaxed) \ - FN(tanh) \ - FN(tanh_relaxed) \ - FN(transpose) \ - FN(transpose_float_1) \ - FN(transpose_float_1_relaxed) \ - FN(transpose_quant8_1) \ - FN(transpose_relaxed) +#define FOR_EACH_TEST_MODEL(FN) FN(random_multinomial) #define FORWARD_DECLARE_GENERATED_OBJECTS(function) \ namespace function { \ diff --git a/neuralnetworks/1.2/vts/functional/ModelsV1_0.h b/neuralnetworks/1.2/vts/functional/ModelsV1_0.h new file mode 100644 index 0000000000..e81e64bb41 --- /dev/null +++ b/neuralnetworks/1.2/vts/functional/ModelsV1_0.h @@ -0,0 +1,202 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef VTS_HAL_NEURALNETWORKS_V1_2_VTS_FUNCTIONAL_MODELS_V1_0_H +#define VTS_HAL_NEURALNETWORKS_V1_2_VTS_FUNCTIONAL_MODELS_V1_0_H + +#define LOG_TAG "neuralnetworks_hidl_hal_test" + +#include "TestHarness.h" + +#include +#include +#include + +namespace android { +namespace hardware { +namespace neuralnetworks { +namespace V1_2 { +namespace vts { +namespace functional { + +using MixedTypedExample = test_helper::MixedTypedExample; + +#define FOR_EACH_TEST_MODEL(FN) \ + FN(add_broadcast_quant8) \ + FN(add) \ + FN(add_quant8) \ + FN(avg_pool_float_1) \ + FN(avg_pool_float_2) \ + FN(avg_pool_float_3) \ + FN(avg_pool_float_4) \ + FN(avg_pool_float_5) \ + FN(avg_pool_quant8_1) \ + FN(avg_pool_quant8_2) \ + FN(avg_pool_quant8_3) \ + FN(avg_pool_quant8_4) \ + FN(avg_pool_quant8_5) \ + FN(concat_float_1) \ + FN(concat_float_2) \ + FN(concat_float_3) \ + FN(concat_quant8_1) \ + FN(concat_quant8_2) \ + FN(concat_quant8_3) \ + FN(conv_1_h3_w2_SAME) \ + FN(conv_1_h3_w2_VALID) \ + FN(conv_3_h3_w2_SAME) \ + FN(conv_3_h3_w2_VALID) \ + FN(conv_float_2) \ + FN(conv_float_channels) \ + FN(conv_float_channels_weights_as_inputs) \ + FN(conv_float_large) \ + FN(conv_float_large_weights_as_inputs) \ + FN(conv_float) \ + FN(conv_float_weights_as_inputs) \ + FN(conv_quant8_2) \ + FN(conv_quant8_channels) \ + FN(conv_quant8_channels_weights_as_inputs) \ + FN(conv_quant8_large) \ + FN(conv_quant8_large_weights_as_inputs) \ + FN(conv_quant8) \ + FN(conv_quant8_overflow) \ + FN(conv_quant8_overflow_weights_as_inputs) \ + FN(conv_quant8_weights_as_inputs) \ + FN(depth_to_space_float_1) \ + FN(depth_to_space_float_2) \ + FN(depth_to_space_float_3) \ + FN(depth_to_space_quant8_1) \ + FN(depth_to_space_quant8_2) \ + FN(depthwise_conv2d_float_2) \ + FN(depthwise_conv2d_float_large_2) \ + FN(depthwise_conv2d_float_large_2_weights_as_inputs) \ + FN(depthwise_conv2d_float_large) \ + FN(depthwise_conv2d_float_large_weights_as_inputs) \ + FN(depthwise_conv2d_float) \ + FN(depthwise_conv2d_float_weights_as_inputs) \ + FN(depthwise_conv2d_quant8_2) \ + FN(depthwise_conv2d_quant8_large) \ + FN(depthwise_conv2d_quant8_large_weights_as_inputs) \ + FN(depthwise_conv2d_quant8) \ + FN(depthwise_conv2d_quant8_weights_as_inputs) \ + FN(depthwise_conv) \ + FN(dequantize) \ + FN(embedding_lookup) \ + FN(floor) \ + FN(fully_connected_float_2) \ + FN(fully_connected_float_large) \ + FN(fully_connected_float_large_weights_as_inputs) \ + FN(fully_connected_float) \ + FN(fully_connected_float_weights_as_inputs) \ + FN(fully_connected_quant8_2) \ + FN(fully_connected_quant8_large) \ + FN(fully_connected_quant8_large_weights_as_inputs) \ + FN(fully_connected_quant8) \ + FN(fully_connected_quant8_weights_as_inputs) \ + FN(hashtable_lookup_float) \ + FN(hashtable_lookup_quant8) \ + FN(l2_normalization_2) \ + FN(l2_normalization_large) \ + FN(l2_normalization) \ + FN(l2_pool_float_2) \ + FN(l2_pool_float_large) \ + FN(l2_pool_float) \ + FN(local_response_norm_float_1) \ + FN(local_response_norm_float_2) \ + FN(local_response_norm_float_3) \ + FN(local_response_norm_float_4) \ + FN(logistic_float_1) \ + FN(logistic_float_2) \ + FN(logistic_quant8_1) \ + FN(logistic_quant8_2) \ + FN(lsh_projection_2) \ + FN(lsh_projection) \ + FN(lsh_projection_weights_as_inputs) \ + FN(lstm2) \ + FN(lstm2_state2) \ + FN(lstm2_state) \ + FN(lstm3) \ + FN(lstm3_state2) \ + FN(lstm3_state3) \ + FN(lstm3_state) \ + FN(lstm) \ + FN(lstm_state2) \ + FN(lstm_state) \ + FN(max_pool_float_1) \ + FN(max_pool_float_2) \ + FN(max_pool_float_3) \ + FN(max_pool_float_4) \ + FN(max_pool_quant8_1) \ + FN(max_pool_quant8_2) \ + FN(max_pool_quant8_3) \ + FN(max_pool_quant8_4) \ + FN(mobilenet_224_gender_basic_fixed) \ + FN(mobilenet_quantized) \ + FN(mul_broadcast_quant8) \ + FN(mul) \ + FN(mul_quant8) \ + FN(mul_relu) \ + FN(relu1_float_1) \ + FN(relu1_float_2) \ + FN(relu1_quant8_1) \ + FN(relu1_quant8_2) \ + FN(relu6_float_1) \ + FN(relu6_float_2) \ + FN(relu6_quant8_1) \ + FN(relu6_quant8_2) \ + FN(relu_float_1) \ + FN(relu_float_2) \ + FN(relu_quant8_1) \ + FN(relu_quant8_2) \ + FN(reshape) \ + FN(reshape_quant8) \ + FN(reshape_quant8_weights_as_inputs) \ + FN(reshape_weights_as_inputs) \ + FN(resize_bilinear_2) \ + FN(resize_bilinear) \ + FN(rnn) \ + FN(rnn_state) \ + FN(softmax_float_1) \ + FN(softmax_float_2) \ + FN(softmax_quant8_1) \ + FN(softmax_quant8_2) \ + FN(space_to_depth_float_1) \ + FN(space_to_depth_float_2) \ + FN(space_to_depth_float_3) \ + FN(space_to_depth_quant8_1) \ + FN(space_to_depth_quant8_2) \ + FN(svdf2) \ + FN(svdf) \ + FN(svdf_state) \ + FN(tanh) + +#define FORWARD_DECLARE_GENERATED_OBJECTS(function) \ + namespace function { \ + extern std::vector examples; \ + Model createTestModel(); \ + } + +FOR_EACH_TEST_MODEL(FORWARD_DECLARE_GENERATED_OBJECTS) + +#undef FORWARD_DECLARE_GENERATED_OBJECTS + +} // namespace functional +} // namespace vts +} // namespace V1_2 +} // namespace neuralnetworks +} // namespace hardware +} // namespace android + +#endif // VTS_HAL_NEURALNETWORKS_V1_2_VTS_FUNCTIONAL_MODELS_V1_0_H diff --git a/neuralnetworks/1.2/vts/functional/ModelsV1_1.h b/neuralnetworks/1.2/vts/functional/ModelsV1_1.h new file mode 100644 index 0000000000..eb68de4459 --- /dev/null +++ b/neuralnetworks/1.2/vts/functional/ModelsV1_1.h @@ -0,0 +1,230 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef VTS_HAL_NEURALNETWORKS_V1_1_VTS_FUNCTIONAL_MODELS_H +#define VTS_HAL_NEURALNETWORKS_V1_1_VTS_FUNCTIONAL_MODELS_H + +#define LOG_TAG "neuralnetworks_hidl_hal_test" + +#include "TestHarness.h" + +#include + +namespace android { +namespace hardware { +namespace neuralnetworks { +namespace V1_2 { +namespace vts { +namespace functional { + +using MixedTypedExample = test_helper::MixedTypedExample; + +#define FOR_EACH_TEST_MODEL(FN) \ + FN(add_relaxed) \ + FN(avg_pool_float_1_relaxed) \ + FN(avg_pool_float_2_relaxed) \ + FN(avg_pool_float_3_relaxed) \ + FN(avg_pool_float_4_relaxed) \ + FN(avg_pool_float_5_relaxed) \ + FN(batch_to_space) \ + FN(batch_to_space_float_1) \ + FN(batch_to_space_float_1_relaxed) \ + FN(batch_to_space_quant8_1) \ + FN(batch_to_space_relaxed) \ + FN(concat_float_1_relaxed) \ + FN(concat_float_2_relaxed) \ + FN(concat_float_3_relaxed) \ + FN(conv_1_h3_w2_SAME_relaxed) \ + FN(conv_1_h3_w2_VALID_relaxed) \ + FN(conv_3_h3_w2_SAME_relaxed) \ + FN(conv_3_h3_w2_VALID_relaxed) \ + FN(conv_float_2_relaxed) \ + FN(conv_float_channels_relaxed) \ + FN(conv_float_channels_weights_as_inputs_relaxed) \ + FN(conv_float_large_relaxed) \ + FN(conv_float_large_weights_as_inputs_relaxed) \ + FN(conv_float_relaxed) \ + FN(conv_float_weights_as_inputs_relaxed) \ + FN(depth_to_space_float_1_relaxed) \ + FN(depth_to_space_float_2_relaxed) \ + FN(depth_to_space_float_3_relaxed) \ + FN(depthwise_conv2d_float_2_relaxed) \ + FN(depthwise_conv2d_float_large_2_relaxed) \ + FN(depthwise_conv2d_float_large_2_weights_as_inputs_relaxed) \ + FN(depthwise_conv2d_float_large_relaxed) \ + FN(depthwise_conv2d_float_large_weights_as_inputs_relaxed) \ + FN(depthwise_conv2d_float_relaxed) \ + FN(depthwise_conv2d_float_weights_as_inputs_relaxed) \ + FN(depthwise_conv_relaxed) \ + FN(dequantize_relaxed) \ + FN(div) \ + FN(div_broadcast_float) \ + FN(div_broadcast_float_relaxed) \ + FN(div_relaxed) \ + FN(embedding_lookup_relaxed) \ + FN(floor_relaxed) \ + FN(fully_connected_float_2_relaxed) \ + FN(fully_connected_float_4d_simple) \ + FN(fully_connected_float_4d_simple_relaxed) \ + FN(fully_connected_float_large_relaxed) \ + FN(fully_connected_float_large_weights_as_inputs_relaxed) \ + FN(fully_connected_float_relaxed) \ + FN(fully_connected_float_weights_as_inputs_relaxed) \ + FN(hashtable_lookup_float_relaxed) \ + FN(l2_normalization_2_relaxed) \ + FN(l2_normalization_large_relaxed) \ + FN(l2_normalization_relaxed) \ + FN(l2_pool_float_2_relaxed) \ + FN(l2_pool_float_large_relaxed) \ + FN(l2_pool_float_relaxed) \ + FN(local_response_norm_float_1_relaxed) \ + FN(local_response_norm_float_2_relaxed) \ + FN(local_response_norm_float_3_relaxed) \ + FN(local_response_norm_float_4_relaxed) \ + FN(logistic_float_1_relaxed) \ + FN(logistic_float_2_relaxed) \ + FN(lsh_projection_2_relaxed) \ + FN(lsh_projection_relaxed) \ + FN(lsh_projection_weights_as_inputs_relaxed) \ + FN(lstm2_relaxed) \ + FN(lstm2_state2_relaxed) \ + FN(lstm2_state_relaxed) \ + FN(lstm3_relaxed) \ + FN(lstm3_state2_relaxed) \ + FN(lstm3_state3_relaxed) \ + FN(lstm3_state_relaxed) \ + FN(lstm_relaxed) \ + FN(lstm_state2_relaxed) \ + FN(lstm_state_relaxed) \ + FN(max_pool_float_1_relaxed) \ + FN(max_pool_float_2_relaxed) \ + FN(max_pool_float_3_relaxed) \ + FN(max_pool_float_4_relaxed) \ + FN(mean) \ + FN(mean_float_1) \ + FN(mean_float_1_relaxed) \ + FN(mean_float_2) \ + FN(mean_float_2_relaxed) \ + FN(mean_quant8_1) \ + FN(mean_quant8_2) \ + FN(mean_relaxed) \ + FN(mobilenet_224_gender_basic_fixed_relaxed) \ + FN(mul_relaxed) \ + FN(mul_relu_relaxed) \ + FN(pad) \ + FN(pad_float_1) \ + FN(pad_float_1_relaxed) \ + FN(pad_relaxed) \ + FN(relu1_float_1_relaxed) \ + FN(relu1_float_2_relaxed) \ + FN(relu6_float_1_relaxed) \ + FN(relu6_float_2_relaxed) \ + FN(relu_float_1_relaxed) \ + FN(relu_float_2_relaxed) \ + FN(reshape_relaxed) \ + FN(reshape_weights_as_inputs_relaxed) \ + FN(resize_bilinear_2_relaxed) \ + FN(resize_bilinear_relaxed) \ + FN(rnn_relaxed) \ + FN(rnn_state_relaxed) \ + FN(softmax_float_1_relaxed) \ + FN(softmax_float_2_relaxed) \ + FN(space_to_batch) \ + FN(space_to_batch_float_1) \ + FN(space_to_batch_float_1_relaxed) \ + FN(space_to_batch_float_2) \ + FN(space_to_batch_float_2_relaxed) \ + FN(space_to_batch_float_3) \ + FN(space_to_batch_float_3_relaxed) \ + FN(space_to_batch_quant8_1) \ + FN(space_to_batch_quant8_2) \ + FN(space_to_batch_quant8_3) \ + FN(space_to_batch_relaxed) \ + FN(space_to_depth_float_1_relaxed) \ + FN(space_to_depth_float_2_relaxed) \ + FN(space_to_depth_float_3_relaxed) \ + FN(squeeze) \ + FN(squeeze_float_1) \ + FN(squeeze_float_1_relaxed) \ + FN(squeeze_quant8_1) \ + FN(squeeze_relaxed) \ + FN(strided_slice) \ + FN(strided_slice_float_1) \ + FN(strided_slice_float_10) \ + FN(strided_slice_float_10_relaxed) \ + FN(strided_slice_float_11) \ + FN(strided_slice_float_11_relaxed) \ + FN(strided_slice_float_1_relaxed) \ + FN(strided_slice_float_2) \ + FN(strided_slice_float_2_relaxed) \ + FN(strided_slice_float_3) \ + FN(strided_slice_float_3_relaxed) \ + FN(strided_slice_float_4) \ + FN(strided_slice_float_4_relaxed) \ + FN(strided_slice_float_5) \ + FN(strided_slice_float_5_relaxed) \ + FN(strided_slice_float_6) \ + FN(strided_slice_float_6_relaxed) \ + FN(strided_slice_float_7) \ + FN(strided_slice_float_7_relaxed) \ + FN(strided_slice_float_8) \ + FN(strided_slice_float_8_relaxed) \ + FN(strided_slice_float_9) \ + FN(strided_slice_float_9_relaxed) \ + FN(strided_slice_qaunt8_10) \ + FN(strided_slice_qaunt8_11) \ + FN(strided_slice_quant8_1) \ + FN(strided_slice_quant8_2) \ + FN(strided_slice_quant8_3) \ + FN(strided_slice_quant8_4) \ + FN(strided_slice_quant8_5) \ + FN(strided_slice_quant8_6) \ + FN(strided_slice_quant8_7) \ + FN(strided_slice_quant8_8) \ + FN(strided_slice_quant8_9) \ + FN(strided_slice_relaxed) \ + FN(sub) \ + FN(sub_broadcast_float) \ + FN(sub_broadcast_float_relaxed) \ + FN(sub_relaxed) \ + FN(svdf2_relaxed) \ + FN(svdf_relaxed) \ + FN(svdf_state_relaxed) \ + FN(tanh_relaxed) \ + FN(transpose) \ + FN(transpose_float_1) \ + FN(transpose_float_1_relaxed) \ + FN(transpose_quant8_1) \ + FN(transpose_relaxed) + +#define FORWARD_DECLARE_GENERATED_OBJECTS(function) \ + namespace function { \ + extern std::vector examples; \ + Model createTestModel(); \ + } + +FOR_EACH_TEST_MODEL(FORWARD_DECLARE_GENERATED_OBJECTS) + +#undef FORWARD_DECLARE_GENERATED_OBJECTS + +} // namespace functional +} // namespace vts +} // namespace V1_2 +} // namespace neuralnetworks +} // namespace hardware +} // namespace android + +#endif // VTS_HAL_NEURALNETWORKS_V1_2_VTS_FUNCTIONAL_MODELS_V1_1_H diff --git a/neuralnetworks/1.2/vts/functional/ValidationTestsV1_0.cpp b/neuralnetworks/1.2/vts/functional/ValidationTestsV1_0.cpp new file mode 100644 index 0000000000..c54ed4370a --- /dev/null +++ b/neuralnetworks/1.2/vts/functional/ValidationTestsV1_0.cpp @@ -0,0 +1,50 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#define LOG_TAG "neuralnetworks_hidl_hal_test" + +#include "ModelsV1_0.h" +#include "VtsHalNeuralnetworks.h" + +namespace android { +namespace hardware { +namespace neuralnetworks { +namespace V1_2 { +namespace vts { +namespace functional { + +// forward declarations +std::vector createRequests(const std::vector& examples); + +// generate validation tests +#define VTS_CURRENT_TEST_CASE(TestName) \ + TEST_F(ValidationTest, TestName) { \ + const Model model = TestName::createTestModel(); \ + const std::vector requests = createRequests(TestName::examples); \ + validateModel(model); \ + validateRequests(model, requests); \ + } + +FOR_EACH_TEST_MODEL(VTS_CURRENT_TEST_CASE) + +#undef VTS_CURRENT_TEST_CASE + +} // namespace functional +} // namespace vts +} // namespace V1_2 +} // namespace neuralnetworks +} // namespace hardware +} // namespace android diff --git a/neuralnetworks/1.2/vts/functional/ValidationTestsV1_1.cpp b/neuralnetworks/1.2/vts/functional/ValidationTestsV1_1.cpp new file mode 100644 index 0000000000..95932d59c9 --- /dev/null +++ b/neuralnetworks/1.2/vts/functional/ValidationTestsV1_1.cpp @@ -0,0 +1,50 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#define LOG_TAG "neuralnetworks_hidl_hal_test" + +#include "ModelsV1_1.h" +#include "VtsHalNeuralnetworks.h" + +namespace android { +namespace hardware { +namespace neuralnetworks { +namespace V1_2 { +namespace vts { +namespace functional { + +// forward declarations +std::vector createRequests(const std::vector& examples); + +// generate validation tests +#define VTS_CURRENT_TEST_CASE(TestName) \ + TEST_F(ValidationTest, TestName) { \ + const Model model = TestName::createTestModel(); \ + const std::vector requests = createRequests(TestName::examples); \ + validateModel(model); \ + validateRequests(model, requests); \ + } + +FOR_EACH_TEST_MODEL(VTS_CURRENT_TEST_CASE) + +#undef VTS_CURRENT_TEST_CASE + +} // namespace functional +} // namespace vts +} // namespace V1_2 +} // namespace neuralnetworks +} // namespace hardware +} // namespace android From a6b1dc7d9bd3411179af4f9edd4eeecbc077e329 Mon Sep 17 00:00:00 2001 From: Xusong Wang Date: Mon, 22 Oct 2018 13:49:00 -0700 Subject: [PATCH 2/6] Fix VTS ValidationTest for 1.2 ops. Skip L2_NORMALIZATION, LOCAL_RESPONSE_NORMALIZATION, and SOFTMAX for addOperationInputTest. Test: VtsHalNeuralnetworksV1_xTargetTest with 1.2 sample driver Change-Id: I16de09e2eef7610e564f1f49ff067af7efca3784 Merged-In: I16de09e2eef7610e564f1f49ff067af7efca3784 (cherry picked from commit 6433728e2ce74d1e06299909229ee4e79badcf09) --- .../1.2/vts/functional/ValidateModel.cpp | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/neuralnetworks/1.2/vts/functional/ValidateModel.cpp b/neuralnetworks/1.2/vts/functional/ValidateModel.cpp index 309602823e..b4921e3305 100644 --- a/neuralnetworks/1.2/vts/functional/ValidateModel.cpp +++ b/neuralnetworks/1.2/vts/functional/ValidateModel.cpp @@ -479,8 +479,22 @@ static void removeOperationOutputTest(const sp& device, const Model& mo ///////////////////////// ADD OPERATION INPUT ///////////////////////// +static bool addOperationInputSkip(const Operation& operation) { + // Skip addOperationInputTest for the following operations. + // L2_NORMALIZATION, LOCAL_RESPONSE_NORMALIZATION, SOFTMAX can have an optional axis parameter. + if (operation.type == OperationType::L2_NORMALIZATION || + operation.type == OperationType::LOCAL_RESPONSE_NORMALIZATION || + operation.type == OperationType::SOFTMAX) { + return true; + } + return false; +} + static void addOperationInputTest(const sp& device, const Model& model) { for (size_t operation = 0; operation < model.operations.size(); ++operation) { + if (addOperationInputSkip(model.operations[operation])) { + continue; + } const std::string message = "addOperationInputTest: operation " + std::to_string(operation); validate(device, message, model, [operation](Model* model) { uint32_t index = addOperand(model, OperandLifeTime::MODEL_INPUT); From a8efddc5584fe39dc30e6d95b9668e58064d5d3c Mon Sep 17 00:00:00 2001 From: "Michael K. Sanders" Date: Fri, 9 Nov 2018 12:10:09 +0000 Subject: [PATCH 3/6] Autogenerates VTS ValidationTest tests. This removes the use of a separately updated list of models that has fallen out of sync. Bug: 119293899 Test: VtsHalNeuralnetworksV1_2TargetTest --hal_service_instance=android.hardware.neuralnetworks@1.2::IDevice/sample-all Test: VtsHalNeuralnetworksV1_2CompatV1_1TargetTest --hal_service_instance=android.hardware.neuralnetworks@1.2::IDevice/sample-all Test: VtsHalNeuralnetworksV1_2CompatV1_0TargetTest --hal_service_instance=android.hardware.neuralnetworks@1.2::IDevice/sample-all Test: VtsHalNeuralnetworksV1_1TargetTest --hal_service_instance=android.hardware.neuralnetworks@1.1::IDevice/sample-all Test: VtsHalNeuralnetworksV1_1CompatV1_0TargetTest --hal_service_instance=android.hardware.neuralnetworks@1.1::IDevice/sample-all Test: VtsHalNeuralnetworksV1_0TargetTest --hal_service_instance=android.hardware.neuralnetworks@1.0::IDevice/sample-all Change-Id: I2d8804d78331b8fceab4c622c871802aa0f0a4b4 Merged-In: I2d8804d78331b8fceab4c622c871802aa0f0a4b4 (cherry picked from commit b5fe58b95a4ab80987d8bae583c77e85de2f8d31) --- neuralnetworks/1.0/vts/functional/Android.bp | 1 - .../1.0/vts/functional/GeneratedTests.cpp | 2 + neuralnetworks/1.0/vts/functional/Models.h | 200 --------------- .../1.0/vts/functional/ValidationTests.cpp | 50 ---- neuralnetworks/1.1/vts/functional/Android.bp | 2 - .../1.1/vts/functional/GeneratedTests.cpp | 2 + .../1.1/vts/functional/GeneratedTestsV1_0.cpp | 2 + neuralnetworks/1.1/vts/functional/Models.h | 231 ------------------ .../1.1/vts/functional/ModelsV1_0.h | 201 --------------- .../1.1/vts/functional/ValidationTests.cpp | 50 ---- .../vts/functional/ValidationTestsV1_0.cpp | 50 ---- neuralnetworks/1.2/vts/functional/Android.bp | 3 - .../1.2/vts/functional/GeneratedTests.cpp | 2 + .../1.2/vts/functional/GeneratedTestsV1_0.cpp | 2 + .../1.2/vts/functional/GeneratedTestsV1_1.cpp | 2 + neuralnetworks/1.2/vts/functional/Models.h | 56 ----- .../1.2/vts/functional/ModelsV1_0.h | 202 --------------- .../1.2/vts/functional/ModelsV1_1.h | 230 ----------------- .../1.2/vts/functional/ValidationTests.cpp | 50 ---- .../vts/functional/ValidationTestsV1_0.cpp | 50 ---- .../vts/functional/ValidationTestsV1_1.cpp | 50 ---- 21 files changed, 12 insertions(+), 1426 deletions(-) delete mode 100644 neuralnetworks/1.0/vts/functional/Models.h delete mode 100644 neuralnetworks/1.0/vts/functional/ValidationTests.cpp delete mode 100644 neuralnetworks/1.1/vts/functional/Models.h delete mode 100644 neuralnetworks/1.1/vts/functional/ModelsV1_0.h delete mode 100644 neuralnetworks/1.1/vts/functional/ValidationTests.cpp delete mode 100644 neuralnetworks/1.1/vts/functional/ValidationTestsV1_0.cpp delete mode 100644 neuralnetworks/1.2/vts/functional/Models.h delete mode 100644 neuralnetworks/1.2/vts/functional/ModelsV1_0.h delete mode 100644 neuralnetworks/1.2/vts/functional/ModelsV1_1.h delete mode 100644 neuralnetworks/1.2/vts/functional/ValidationTests.cpp delete mode 100644 neuralnetworks/1.2/vts/functional/ValidationTestsV1_0.cpp delete mode 100644 neuralnetworks/1.2/vts/functional/ValidationTestsV1_1.cpp diff --git a/neuralnetworks/1.0/vts/functional/Android.bp b/neuralnetworks/1.0/vts/functional/Android.bp index 1b5e338bed..234527a743 100644 --- a/neuralnetworks/1.0/vts/functional/Android.bp +++ b/neuralnetworks/1.0/vts/functional/Android.bp @@ -80,6 +80,5 @@ cc_test { srcs: [ "BasicTests.cpp", "GeneratedTests.cpp", - "ValidationTests.cpp", ], } diff --git a/neuralnetworks/1.0/vts/functional/GeneratedTests.cpp b/neuralnetworks/1.0/vts/functional/GeneratedTests.cpp index ac1ae60d85..26b4d8b002 100644 --- a/neuralnetworks/1.0/vts/functional/GeneratedTests.cpp +++ b/neuralnetworks/1.0/vts/functional/GeneratedTests.cpp @@ -45,6 +45,8 @@ using ::android::hardware::neuralnetworks::V1_0::implementation::PreparedModelCa using ::android::nn::allocateSharedMemory; using ::test_helper::MixedTypedExample; +std::vector createRequests(const std::vector& examples); + // in frameworks/ml/nn/runtime/tests/generated/ #include "all_generated_V1_0_vts_tests.cpp" diff --git a/neuralnetworks/1.0/vts/functional/Models.h b/neuralnetworks/1.0/vts/functional/Models.h deleted file mode 100644 index 268e67159f..0000000000 --- a/neuralnetworks/1.0/vts/functional/Models.h +++ /dev/null @@ -1,200 +0,0 @@ -/* - * Copyright (C) 2018 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef VTS_HAL_NEURALNETWORKS_V1_0_VTS_FUNCTIONAL_MODELS_H -#define VTS_HAL_NEURALNETWORKS_V1_0_VTS_FUNCTIONAL_MODELS_H - -#define LOG_TAG "neuralnetworks_hidl_hal_test" - -#include "TestHarness.h" - -#include - -namespace android { -namespace hardware { -namespace neuralnetworks { -namespace V1_0 { -namespace vts { -namespace functional { - -using MixedTypedExample = test_helper::MixedTypedExample; - -#define FOR_EACH_TEST_MODEL(FN) \ - FN(add_broadcast_quant8) \ - FN(add) \ - FN(add_quant8) \ - FN(avg_pool_float_1) \ - FN(avg_pool_float_2) \ - FN(avg_pool_float_3) \ - FN(avg_pool_float_4) \ - FN(avg_pool_float_5) \ - FN(avg_pool_quant8_1) \ - FN(avg_pool_quant8_2) \ - FN(avg_pool_quant8_3) \ - FN(avg_pool_quant8_4) \ - FN(avg_pool_quant8_5) \ - FN(concat_float_1) \ - FN(concat_float_2) \ - FN(concat_float_3) \ - FN(concat_quant8_1) \ - FN(concat_quant8_2) \ - FN(concat_quant8_3) \ - FN(conv_1_h3_w2_SAME) \ - FN(conv_1_h3_w2_VALID) \ - FN(conv_3_h3_w2_SAME) \ - FN(conv_3_h3_w2_VALID) \ - FN(conv_float_2) \ - FN(conv_float_channels) \ - FN(conv_float_channels_weights_as_inputs) \ - FN(conv_float_large) \ - FN(conv_float_large_weights_as_inputs) \ - FN(conv_float) \ - FN(conv_float_weights_as_inputs) \ - FN(conv_quant8_2) \ - FN(conv_quant8_channels) \ - FN(conv_quant8_channels_weights_as_inputs) \ - FN(conv_quant8_large) \ - FN(conv_quant8_large_weights_as_inputs) \ - FN(conv_quant8) \ - FN(conv_quant8_overflow) \ - FN(conv_quant8_overflow_weights_as_inputs) \ - FN(conv_quant8_weights_as_inputs) \ - FN(depth_to_space_float_1) \ - FN(depth_to_space_float_2) \ - FN(depth_to_space_float_3) \ - FN(depth_to_space_quant8_1) \ - FN(depth_to_space_quant8_2) \ - FN(depthwise_conv2d_float_2) \ - FN(depthwise_conv2d_float_large_2) \ - FN(depthwise_conv2d_float_large_2_weights_as_inputs) \ - FN(depthwise_conv2d_float_large) \ - FN(depthwise_conv2d_float_large_weights_as_inputs) \ - FN(depthwise_conv2d_float) \ - FN(depthwise_conv2d_float_weights_as_inputs) \ - FN(depthwise_conv2d_quant8_2) \ - FN(depthwise_conv2d_quant8_large) \ - FN(depthwise_conv2d_quant8_large_weights_as_inputs) \ - FN(depthwise_conv2d_quant8) \ - FN(depthwise_conv2d_quant8_weights_as_inputs) \ - FN(depthwise_conv) \ - FN(dequantize) \ - FN(embedding_lookup) \ - FN(floor) \ - FN(fully_connected_float_2) \ - FN(fully_connected_float_large) \ - FN(fully_connected_float_large_weights_as_inputs) \ - FN(fully_connected_float) \ - FN(fully_connected_float_weights_as_inputs) \ - FN(fully_connected_quant8_2) \ - FN(fully_connected_quant8_large) \ - FN(fully_connected_quant8_large_weights_as_inputs) \ - FN(fully_connected_quant8) \ - FN(fully_connected_quant8_weights_as_inputs) \ - FN(hashtable_lookup_float) \ - FN(hashtable_lookup_quant8) \ - FN(l2_normalization_2) \ - FN(l2_normalization_large) \ - FN(l2_normalization) \ - FN(l2_pool_float_2) \ - FN(l2_pool_float_large) \ - FN(l2_pool_float) \ - FN(local_response_norm_float_1) \ - FN(local_response_norm_float_2) \ - FN(local_response_norm_float_3) \ - FN(local_response_norm_float_4) \ - FN(logistic_float_1) \ - FN(logistic_float_2) \ - FN(logistic_quant8_1) \ - FN(logistic_quant8_2) \ - FN(lsh_projection_2) \ - FN(lsh_projection) \ - FN(lsh_projection_weights_as_inputs) \ - FN(lstm2) \ - FN(lstm2_state2) \ - FN(lstm2_state) \ - FN(lstm3) \ - FN(lstm3_state2) \ - FN(lstm3_state3) \ - FN(lstm3_state) \ - FN(lstm) \ - FN(lstm_state2) \ - FN(lstm_state) \ - FN(max_pool_float_1) \ - FN(max_pool_float_2) \ - FN(max_pool_float_3) \ - FN(max_pool_float_4) \ - FN(max_pool_quant8_1) \ - FN(max_pool_quant8_2) \ - FN(max_pool_quant8_3) \ - FN(max_pool_quant8_4) \ - FN(mobilenet_224_gender_basic_fixed) \ - FN(mobilenet_quantized) \ - FN(mul_broadcast_quant8) \ - FN(mul) \ - FN(mul_quant8) \ - FN(mul_relu) \ - FN(relu1_float_1) \ - FN(relu1_float_2) \ - FN(relu1_quant8_1) \ - FN(relu1_quant8_2) \ - FN(relu6_float_1) \ - FN(relu6_float_2) \ - FN(relu6_quant8_1) \ - FN(relu6_quant8_2) \ - FN(relu_float_1) \ - FN(relu_float_2) \ - FN(relu_quant8_1) \ - FN(relu_quant8_2) \ - FN(reshape) \ - FN(reshape_quant8) \ - FN(reshape_quant8_weights_as_inputs) \ - FN(reshape_weights_as_inputs) \ - FN(resize_bilinear_2) \ - FN(resize_bilinear) \ - FN(rnn) \ - FN(rnn_state) \ - FN(softmax_float_1) \ - FN(softmax_float_2) \ - FN(softmax_quant8_1) \ - FN(softmax_quant8_2) \ - FN(space_to_depth_float_1) \ - FN(space_to_depth_float_2) \ - FN(space_to_depth_float_3) \ - FN(space_to_depth_quant8_1) \ - FN(space_to_depth_quant8_2) \ - FN(svdf2) \ - FN(svdf) \ - FN(svdf_state) \ - FN(tanh) - -#define FORWARD_DECLARE_GENERATED_OBJECTS(function) \ - namespace function { \ - extern std::vector examples; \ - Model createTestModel(); \ - } - -FOR_EACH_TEST_MODEL(FORWARD_DECLARE_GENERATED_OBJECTS) - -#undef FORWARD_DECLARE_GENERATED_OBJECTS - -} // namespace functional -} // namespace vts -} // namespace V1_0 -} // namespace neuralnetworks -} // namespace hardware -} // namespace android - -#endif // VTS_HAL_NEURALNETWORKS_V1_0_VTS_FUNCTIONAL_MODELS_H diff --git a/neuralnetworks/1.0/vts/functional/ValidationTests.cpp b/neuralnetworks/1.0/vts/functional/ValidationTests.cpp deleted file mode 100644 index d3cbcff0a7..0000000000 --- a/neuralnetworks/1.0/vts/functional/ValidationTests.cpp +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright (C) 2018 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#define LOG_TAG "neuralnetworks_hidl_hal_test" - -#include "Models.h" -#include "VtsHalNeuralnetworks.h" - -namespace android { -namespace hardware { -namespace neuralnetworks { -namespace V1_0 { -namespace vts { -namespace functional { - -// forward declarations -std::vector createRequests(const std::vector<::test_helper::MixedTypedExample>& examples); - -// generate validation tests -#define VTS_CURRENT_TEST_CASE(TestName) \ - TEST_F(ValidationTest, TestName) { \ - const Model model = TestName::createTestModel(); \ - const std::vector requests = createRequests(TestName::examples); \ - validateModel(model); \ - validateRequests(model, requests); \ - } - -FOR_EACH_TEST_MODEL(VTS_CURRENT_TEST_CASE) - -#undef VTS_CURRENT_TEST_CASE - -} // namespace functional -} // namespace vts -} // namespace V1_0 -} // namespace neuralnetworks -} // namespace hardware -} // namespace android diff --git a/neuralnetworks/1.1/vts/functional/Android.bp b/neuralnetworks/1.1/vts/functional/Android.bp index df1ac67927..07c9b6ebbc 100644 --- a/neuralnetworks/1.1/vts/functional/Android.bp +++ b/neuralnetworks/1.1/vts/functional/Android.bp @@ -20,7 +20,6 @@ cc_test { defaults: ["VtsHalNeuralNetworksTargetTestDefaults"], srcs: [ "GeneratedTestsV1_0.cpp", - "ValidationTestsV1_0.cpp", ], } @@ -31,6 +30,5 @@ cc_test { srcs: [ "BasicTests.cpp", "GeneratedTests.cpp", - "ValidationTests.cpp", ], } diff --git a/neuralnetworks/1.1/vts/functional/GeneratedTests.cpp b/neuralnetworks/1.1/vts/functional/GeneratedTests.cpp index d16f181d12..290a9d33a3 100644 --- a/neuralnetworks/1.1/vts/functional/GeneratedTests.cpp +++ b/neuralnetworks/1.1/vts/functional/GeneratedTests.cpp @@ -45,6 +45,8 @@ using ::android::hardware::neuralnetworks::V1_0::implementation::PreparedModelCa using ::android::nn::allocateSharedMemory; using ::test_helper::MixedTypedExample; +std::vector createRequests(const std::vector& examples); + // in frameworks/ml/nn/runtime/tests/generated/ #include "all_generated_V1_1_vts_tests.cpp" diff --git a/neuralnetworks/1.1/vts/functional/GeneratedTestsV1_0.cpp b/neuralnetworks/1.1/vts/functional/GeneratedTestsV1_0.cpp index e2acd7ded4..a36b24ceb0 100644 --- a/neuralnetworks/1.1/vts/functional/GeneratedTestsV1_0.cpp +++ b/neuralnetworks/1.1/vts/functional/GeneratedTestsV1_0.cpp @@ -45,6 +45,8 @@ using ::android::hardware::neuralnetworks::V1_0::implementation::PreparedModelCa using ::android::nn::allocateSharedMemory; using ::test_helper::MixedTypedExample; +std::vector createRequests(const std::vector& examples); + // in frameworks/ml/nn/runtime/tests/generated/ #include "all_generated_V1_0_vts_tests.cpp" diff --git a/neuralnetworks/1.1/vts/functional/Models.h b/neuralnetworks/1.1/vts/functional/Models.h deleted file mode 100644 index 57da0102da..0000000000 --- a/neuralnetworks/1.1/vts/functional/Models.h +++ /dev/null @@ -1,231 +0,0 @@ -/* - * Copyright (C) 2018 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef VTS_HAL_NEURALNETWORKS_V1_1_VTS_FUNCTIONAL_MODELS_H -#define VTS_HAL_NEURALNETWORKS_V1_1_VTS_FUNCTIONAL_MODELS_H - -#define LOG_TAG "neuralnetworks_hidl_hal_test" - -#include "TestHarness.h" - -#include -#include - -namespace android { -namespace hardware { -namespace neuralnetworks { -namespace V1_1 { -namespace vts { -namespace functional { - -using MixedTypedExample = test_helper::MixedTypedExample; - -#define FOR_EACH_TEST_MODEL(FN) \ - FN(add_relaxed) \ - FN(avg_pool_float_1_relaxed) \ - FN(avg_pool_float_2_relaxed) \ - FN(avg_pool_float_3_relaxed) \ - FN(avg_pool_float_4_relaxed) \ - FN(avg_pool_float_5_relaxed) \ - FN(batch_to_space) \ - FN(batch_to_space_float_1) \ - FN(batch_to_space_float_1_relaxed) \ - FN(batch_to_space_quant8_1) \ - FN(batch_to_space_relaxed) \ - FN(concat_float_1_relaxed) \ - FN(concat_float_2_relaxed) \ - FN(concat_float_3_relaxed) \ - FN(conv_1_h3_w2_SAME_relaxed) \ - FN(conv_1_h3_w2_VALID_relaxed) \ - FN(conv_3_h3_w2_SAME_relaxed) \ - FN(conv_3_h3_w2_VALID_relaxed) \ - FN(conv_float_2_relaxed) \ - FN(conv_float_channels_relaxed) \ - FN(conv_float_channels_weights_as_inputs_relaxed) \ - FN(conv_float_large_relaxed) \ - FN(conv_float_large_weights_as_inputs_relaxed) \ - FN(conv_float_relaxed) \ - FN(conv_float_weights_as_inputs_relaxed) \ - FN(depth_to_space_float_1_relaxed) \ - FN(depth_to_space_float_2_relaxed) \ - FN(depth_to_space_float_3_relaxed) \ - FN(depthwise_conv2d_float_2_relaxed) \ - FN(depthwise_conv2d_float_large_2_relaxed) \ - FN(depthwise_conv2d_float_large_2_weights_as_inputs_relaxed) \ - FN(depthwise_conv2d_float_large_relaxed) \ - FN(depthwise_conv2d_float_large_weights_as_inputs_relaxed) \ - FN(depthwise_conv2d_float_relaxed) \ - FN(depthwise_conv2d_float_weights_as_inputs_relaxed) \ - FN(depthwise_conv_relaxed) \ - FN(dequantize_relaxed) \ - FN(div) \ - FN(div_broadcast_float) \ - FN(div_broadcast_float_relaxed) \ - FN(div_relaxed) \ - FN(embedding_lookup_relaxed) \ - FN(floor_relaxed) \ - FN(fully_connected_float_2_relaxed) \ - FN(fully_connected_float_4d_simple) \ - FN(fully_connected_float_4d_simple_relaxed) \ - FN(fully_connected_float_large_relaxed) \ - FN(fully_connected_float_large_weights_as_inputs_relaxed) \ - FN(fully_connected_float_relaxed) \ - FN(fully_connected_float_weights_as_inputs_relaxed) \ - FN(hashtable_lookup_float_relaxed) \ - FN(l2_normalization_2_relaxed) \ - FN(l2_normalization_large_relaxed) \ - FN(l2_normalization_relaxed) \ - FN(l2_pool_float_2_relaxed) \ - FN(l2_pool_float_large_relaxed) \ - FN(l2_pool_float_relaxed) \ - FN(local_response_norm_float_1_relaxed) \ - FN(local_response_norm_float_2_relaxed) \ - FN(local_response_norm_float_3_relaxed) \ - FN(local_response_norm_float_4_relaxed) \ - FN(logistic_float_1_relaxed) \ - FN(logistic_float_2_relaxed) \ - FN(lsh_projection_2_relaxed) \ - FN(lsh_projection_relaxed) \ - FN(lsh_projection_weights_as_inputs_relaxed) \ - FN(lstm2_relaxed) \ - FN(lstm2_state2_relaxed) \ - FN(lstm2_state_relaxed) \ - FN(lstm3_relaxed) \ - FN(lstm3_state2_relaxed) \ - FN(lstm3_state3_relaxed) \ - FN(lstm3_state_relaxed) \ - FN(lstm_relaxed) \ - FN(lstm_state2_relaxed) \ - FN(lstm_state_relaxed) \ - FN(max_pool_float_1_relaxed) \ - FN(max_pool_float_2_relaxed) \ - FN(max_pool_float_3_relaxed) \ - FN(max_pool_float_4_relaxed) \ - FN(mean) \ - FN(mean_float_1) \ - FN(mean_float_1_relaxed) \ - FN(mean_float_2) \ - FN(mean_float_2_relaxed) \ - FN(mean_quant8_1) \ - FN(mean_quant8_2) \ - FN(mean_relaxed) \ - FN(mobilenet_224_gender_basic_fixed_relaxed) \ - FN(mul_relaxed) \ - FN(mul_relu_relaxed) \ - FN(pad) \ - FN(pad_float_1) \ - FN(pad_float_1_relaxed) \ - FN(pad_relaxed) \ - FN(relu1_float_1_relaxed) \ - FN(relu1_float_2_relaxed) \ - FN(relu6_float_1_relaxed) \ - FN(relu6_float_2_relaxed) \ - FN(relu_float_1_relaxed) \ - FN(relu_float_2_relaxed) \ - FN(reshape_relaxed) \ - FN(reshape_weights_as_inputs_relaxed) \ - FN(resize_bilinear_2_relaxed) \ - FN(resize_bilinear_relaxed) \ - FN(rnn_relaxed) \ - FN(rnn_state_relaxed) \ - FN(softmax_float_1_relaxed) \ - FN(softmax_float_2_relaxed) \ - FN(space_to_batch) \ - FN(space_to_batch_float_1) \ - FN(space_to_batch_float_1_relaxed) \ - FN(space_to_batch_float_2) \ - FN(space_to_batch_float_2_relaxed) \ - FN(space_to_batch_float_3) \ - FN(space_to_batch_float_3_relaxed) \ - FN(space_to_batch_quant8_1) \ - FN(space_to_batch_quant8_2) \ - FN(space_to_batch_quant8_3) \ - FN(space_to_batch_relaxed) \ - FN(space_to_depth_float_1_relaxed) \ - FN(space_to_depth_float_2_relaxed) \ - FN(space_to_depth_float_3_relaxed) \ - FN(squeeze) \ - FN(squeeze_float_1) \ - FN(squeeze_float_1_relaxed) \ - FN(squeeze_quant8_1) \ - FN(squeeze_relaxed) \ - FN(strided_slice) \ - FN(strided_slice_float_1) \ - FN(strided_slice_float_10) \ - FN(strided_slice_float_10_relaxed) \ - FN(strided_slice_float_11) \ - FN(strided_slice_float_11_relaxed) \ - FN(strided_slice_float_1_relaxed) \ - FN(strided_slice_float_2) \ - FN(strided_slice_float_2_relaxed) \ - FN(strided_slice_float_3) \ - FN(strided_slice_float_3_relaxed) \ - FN(strided_slice_float_4) \ - FN(strided_slice_float_4_relaxed) \ - FN(strided_slice_float_5) \ - FN(strided_slice_float_5_relaxed) \ - FN(strided_slice_float_6) \ - FN(strided_slice_float_6_relaxed) \ - FN(strided_slice_float_7) \ - FN(strided_slice_float_7_relaxed) \ - FN(strided_slice_float_8) \ - FN(strided_slice_float_8_relaxed) \ - FN(strided_slice_float_9) \ - FN(strided_slice_float_9_relaxed) \ - FN(strided_slice_qaunt8_10) \ - FN(strided_slice_qaunt8_11) \ - FN(strided_slice_quant8_1) \ - FN(strided_slice_quant8_2) \ - FN(strided_slice_quant8_3) \ - FN(strided_slice_quant8_4) \ - FN(strided_slice_quant8_5) \ - FN(strided_slice_quant8_6) \ - FN(strided_slice_quant8_7) \ - FN(strided_slice_quant8_8) \ - FN(strided_slice_quant8_9) \ - FN(strided_slice_relaxed) \ - FN(sub) \ - FN(sub_broadcast_float) \ - FN(sub_broadcast_float_relaxed) \ - FN(sub_relaxed) \ - FN(svdf2_relaxed) \ - FN(svdf_relaxed) \ - FN(svdf_state_relaxed) \ - FN(tanh_relaxed) \ - FN(transpose) \ - FN(transpose_float_1) \ - FN(transpose_float_1_relaxed) \ - FN(transpose_quant8_1) \ - FN(transpose_relaxed) - -#define FORWARD_DECLARE_GENERATED_OBJECTS(function) \ - namespace function { \ - extern std::vector examples; \ - Model createTestModel(); \ - } - -FOR_EACH_TEST_MODEL(FORWARD_DECLARE_GENERATED_OBJECTS) - -#undef FORWARD_DECLARE_GENERATED_OBJECTS - -} // namespace functional -} // namespace vts -} // namespace V1_1 -} // namespace neuralnetworks -} // namespace hardware -} // namespace android - -#endif // VTS_HAL_NEURALNETWORKS_V1_1_VTS_FUNCTIONAL_MODELS_H diff --git a/neuralnetworks/1.1/vts/functional/ModelsV1_0.h b/neuralnetworks/1.1/vts/functional/ModelsV1_0.h deleted file mode 100644 index 52c0346aaa..0000000000 --- a/neuralnetworks/1.1/vts/functional/ModelsV1_0.h +++ /dev/null @@ -1,201 +0,0 @@ -/* - * Copyright (C) 2018 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef VTS_HAL_NEURALNETWORKS_V1_1_VTS_FUNCTIONAL_MODELS_V1_0_H -#define VTS_HAL_NEURALNETWORKS_V1_1_VTS_FUNCTIONAL_MODELS_V1_0_H - -#define LOG_TAG "neuralnetworks_hidl_hal_test" - -#include "TestHarness.h" - -#include -#include - -namespace android { -namespace hardware { -namespace neuralnetworks { -namespace V1_1 { -namespace vts { -namespace functional { - -using MixedTypedExample = test_helper::MixedTypedExample; - -#define FOR_EACH_TEST_MODEL(FN) \ - FN(add_broadcast_quant8) \ - FN(add) \ - FN(add_quant8) \ - FN(avg_pool_float_1) \ - FN(avg_pool_float_2) \ - FN(avg_pool_float_3) \ - FN(avg_pool_float_4) \ - FN(avg_pool_float_5) \ - FN(avg_pool_quant8_1) \ - FN(avg_pool_quant8_2) \ - FN(avg_pool_quant8_3) \ - FN(avg_pool_quant8_4) \ - FN(avg_pool_quant8_5) \ - FN(concat_float_1) \ - FN(concat_float_2) \ - FN(concat_float_3) \ - FN(concat_quant8_1) \ - FN(concat_quant8_2) \ - FN(concat_quant8_3) \ - FN(conv_1_h3_w2_SAME) \ - FN(conv_1_h3_w2_VALID) \ - FN(conv_3_h3_w2_SAME) \ - FN(conv_3_h3_w2_VALID) \ - FN(conv_float_2) \ - FN(conv_float_channels) \ - FN(conv_float_channels_weights_as_inputs) \ - FN(conv_float_large) \ - FN(conv_float_large_weights_as_inputs) \ - FN(conv_float) \ - FN(conv_float_weights_as_inputs) \ - FN(conv_quant8_2) \ - FN(conv_quant8_channels) \ - FN(conv_quant8_channels_weights_as_inputs) \ - FN(conv_quant8_large) \ - FN(conv_quant8_large_weights_as_inputs) \ - FN(conv_quant8) \ - FN(conv_quant8_overflow) \ - FN(conv_quant8_overflow_weights_as_inputs) \ - FN(conv_quant8_weights_as_inputs) \ - FN(depth_to_space_float_1) \ - FN(depth_to_space_float_2) \ - FN(depth_to_space_float_3) \ - FN(depth_to_space_quant8_1) \ - FN(depth_to_space_quant8_2) \ - FN(depthwise_conv2d_float_2) \ - FN(depthwise_conv2d_float_large_2) \ - FN(depthwise_conv2d_float_large_2_weights_as_inputs) \ - FN(depthwise_conv2d_float_large) \ - FN(depthwise_conv2d_float_large_weights_as_inputs) \ - FN(depthwise_conv2d_float) \ - FN(depthwise_conv2d_float_weights_as_inputs) \ - FN(depthwise_conv2d_quant8_2) \ - FN(depthwise_conv2d_quant8_large) \ - FN(depthwise_conv2d_quant8_large_weights_as_inputs) \ - FN(depthwise_conv2d_quant8) \ - FN(depthwise_conv2d_quant8_weights_as_inputs) \ - FN(depthwise_conv) \ - FN(dequantize) \ - FN(embedding_lookup) \ - FN(floor) \ - FN(fully_connected_float_2) \ - FN(fully_connected_float_large) \ - FN(fully_connected_float_large_weights_as_inputs) \ - FN(fully_connected_float) \ - FN(fully_connected_float_weights_as_inputs) \ - FN(fully_connected_quant8_2) \ - FN(fully_connected_quant8_large) \ - FN(fully_connected_quant8_large_weights_as_inputs) \ - FN(fully_connected_quant8) \ - FN(fully_connected_quant8_weights_as_inputs) \ - FN(hashtable_lookup_float) \ - FN(hashtable_lookup_quant8) \ - FN(l2_normalization_2) \ - FN(l2_normalization_large) \ - FN(l2_normalization) \ - FN(l2_pool_float_2) \ - FN(l2_pool_float_large) \ - FN(l2_pool_float) \ - FN(local_response_norm_float_1) \ - FN(local_response_norm_float_2) \ - FN(local_response_norm_float_3) \ - FN(local_response_norm_float_4) \ - FN(logistic_float_1) \ - FN(logistic_float_2) \ - FN(logistic_quant8_1) \ - FN(logistic_quant8_2) \ - FN(lsh_projection_2) \ - FN(lsh_projection) \ - FN(lsh_projection_weights_as_inputs) \ - FN(lstm2) \ - FN(lstm2_state2) \ - FN(lstm2_state) \ - FN(lstm3) \ - FN(lstm3_state2) \ - FN(lstm3_state3) \ - FN(lstm3_state) \ - FN(lstm) \ - FN(lstm_state2) \ - FN(lstm_state) \ - FN(max_pool_float_1) \ - FN(max_pool_float_2) \ - FN(max_pool_float_3) \ - FN(max_pool_float_4) \ - FN(max_pool_quant8_1) \ - FN(max_pool_quant8_2) \ - FN(max_pool_quant8_3) \ - FN(max_pool_quant8_4) \ - FN(mobilenet_224_gender_basic_fixed) \ - FN(mobilenet_quantized) \ - FN(mul_broadcast_quant8) \ - FN(mul) \ - FN(mul_quant8) \ - FN(mul_relu) \ - FN(relu1_float_1) \ - FN(relu1_float_2) \ - FN(relu1_quant8_1) \ - FN(relu1_quant8_2) \ - FN(relu6_float_1) \ - FN(relu6_float_2) \ - FN(relu6_quant8_1) \ - FN(relu6_quant8_2) \ - FN(relu_float_1) \ - FN(relu_float_2) \ - FN(relu_quant8_1) \ - FN(relu_quant8_2) \ - FN(reshape) \ - FN(reshape_quant8) \ - FN(reshape_quant8_weights_as_inputs) \ - FN(reshape_weights_as_inputs) \ - FN(resize_bilinear_2) \ - FN(resize_bilinear) \ - FN(rnn) \ - FN(rnn_state) \ - FN(softmax_float_1) \ - FN(softmax_float_2) \ - FN(softmax_quant8_1) \ - FN(softmax_quant8_2) \ - FN(space_to_depth_float_1) \ - FN(space_to_depth_float_2) \ - FN(space_to_depth_float_3) \ - FN(space_to_depth_quant8_1) \ - FN(space_to_depth_quant8_2) \ - FN(svdf2) \ - FN(svdf) \ - FN(svdf_state) \ - FN(tanh) - -#define FORWARD_DECLARE_GENERATED_OBJECTS(function) \ - namespace function { \ - extern std::vector examples; \ - Model createTestModel(); \ - } - -FOR_EACH_TEST_MODEL(FORWARD_DECLARE_GENERATED_OBJECTS) - -#undef FORWARD_DECLARE_GENERATED_OBJECTS - -} // namespace functional -} // namespace vts -} // namespace V1_1 -} // namespace neuralnetworks -} // namespace hardware -} // namespace android - -#endif // VTS_HAL_NEURALNETWORKS_V1_1_VTS_FUNCTIONAL_MODELS_V1_0_H diff --git a/neuralnetworks/1.1/vts/functional/ValidationTests.cpp b/neuralnetworks/1.1/vts/functional/ValidationTests.cpp deleted file mode 100644 index 1c35ba842b..0000000000 --- a/neuralnetworks/1.1/vts/functional/ValidationTests.cpp +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright (C) 2018 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#define LOG_TAG "neuralnetworks_hidl_hal_test" - -#include "Models.h" -#include "VtsHalNeuralnetworks.h" - -namespace android { -namespace hardware { -namespace neuralnetworks { -namespace V1_1 { -namespace vts { -namespace functional { - -// forward declarations -std::vector createRequests(const std::vector& examples); - -// generate validation tests -#define VTS_CURRENT_TEST_CASE(TestName) \ - TEST_F(ValidationTest, TestName) { \ - const Model model = TestName::createTestModel(); \ - const std::vector requests = createRequests(TestName::examples); \ - validateModel(model); \ - validateRequests(model, requests); \ - } - -FOR_EACH_TEST_MODEL(VTS_CURRENT_TEST_CASE) - -#undef VTS_CURRENT_TEST_CASE - -} // namespace functional -} // namespace vts -} // namespace V1_1 -} // namespace neuralnetworks -} // namespace hardware -} // namespace android diff --git a/neuralnetworks/1.1/vts/functional/ValidationTestsV1_0.cpp b/neuralnetworks/1.1/vts/functional/ValidationTestsV1_0.cpp deleted file mode 100644 index 7e2af05501..0000000000 --- a/neuralnetworks/1.1/vts/functional/ValidationTestsV1_0.cpp +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright (C) 2018 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#define LOG_TAG "neuralnetworks_hidl_hal_test" - -#include "ModelsV1_0.h" -#include "VtsHalNeuralnetworks.h" - -namespace android { -namespace hardware { -namespace neuralnetworks { -namespace V1_1 { -namespace vts { -namespace functional { - -// forward declarations -std::vector createRequests(const std::vector& examples); - -// generate validation tests -#define VTS_CURRENT_TEST_CASE(TestName) \ - TEST_F(ValidationTest, TestName) { \ - const Model model = TestName::createTestModel(); \ - const std::vector requests = createRequests(TestName::examples); \ - validateModel(model); \ - validateRequests(model, requests); \ - } - -FOR_EACH_TEST_MODEL(VTS_CURRENT_TEST_CASE) - -#undef VTS_CURRENT_TEST_CASE - -} // namespace functional -} // namespace vts -} // namespace V1_1 -} // namespace neuralnetworks -} // namespace hardware -} // namespace android diff --git a/neuralnetworks/1.2/vts/functional/Android.bp b/neuralnetworks/1.2/vts/functional/Android.bp index 087c12f811..085d5db22a 100644 --- a/neuralnetworks/1.2/vts/functional/Android.bp +++ b/neuralnetworks/1.2/vts/functional/Android.bp @@ -20,7 +20,6 @@ cc_test { defaults: ["VtsHalNeuralNetworksTargetTestDefaults"], srcs: [ "GeneratedTestsV1_0.cpp", - "ValidationTestsV1_0.cpp", ] } @@ -30,7 +29,6 @@ cc_test { defaults: ["VtsHalNeuralNetworksTargetTestDefaults"], srcs: [ "GeneratedTestsV1_1.cpp", - "ValidationTestsV1_1.cpp", ], } @@ -41,6 +39,5 @@ cc_test { srcs: [ "BasicTests.cpp", "GeneratedTests.cpp", - "ValidationTests.cpp", ], } diff --git a/neuralnetworks/1.2/vts/functional/GeneratedTests.cpp b/neuralnetworks/1.2/vts/functional/GeneratedTests.cpp index 0608139afc..79d5a606b2 100644 --- a/neuralnetworks/1.2/vts/functional/GeneratedTests.cpp +++ b/neuralnetworks/1.2/vts/functional/GeneratedTests.cpp @@ -45,6 +45,8 @@ using ::android::hardware::neuralnetworks::V1_0::implementation::PreparedModelCa using ::android::nn::allocateSharedMemory; using ::test_helper::MixedTypedExample; +std::vector createRequests(const std::vector& examples); + // in frameworks/ml/nn/runtime/tests/generated/ #include "all_generated_V1_2_vts_tests.cpp" diff --git a/neuralnetworks/1.2/vts/functional/GeneratedTestsV1_0.cpp b/neuralnetworks/1.2/vts/functional/GeneratedTestsV1_0.cpp index 8d685d190d..42e22b03a9 100644 --- a/neuralnetworks/1.2/vts/functional/GeneratedTestsV1_0.cpp +++ b/neuralnetworks/1.2/vts/functional/GeneratedTestsV1_0.cpp @@ -45,6 +45,8 @@ using ::android::hardware::neuralnetworks::V1_0::implementation::PreparedModelCa using ::android::nn::allocateSharedMemory; using ::test_helper::MixedTypedExample; +std::vector createRequests(const std::vector& examples); + // in frameworks/ml/nn/runtime/tests/generated/ #include "all_generated_V1_0_vts_tests.cpp" diff --git a/neuralnetworks/1.2/vts/functional/GeneratedTestsV1_1.cpp b/neuralnetworks/1.2/vts/functional/GeneratedTestsV1_1.cpp index 8dbb586b87..aab5cb6f94 100644 --- a/neuralnetworks/1.2/vts/functional/GeneratedTestsV1_1.cpp +++ b/neuralnetworks/1.2/vts/functional/GeneratedTestsV1_1.cpp @@ -45,6 +45,8 @@ using ::android::hardware::neuralnetworks::V1_0::implementation::PreparedModelCa using ::android::nn::allocateSharedMemory; using ::test_helper::MixedTypedExample; +std::vector createRequests(const std::vector& examples); + // in frameworks/ml/nn/runtime/tests/generated/ #include "all_generated_V1_1_vts_tests.cpp" diff --git a/neuralnetworks/1.2/vts/functional/Models.h b/neuralnetworks/1.2/vts/functional/Models.h deleted file mode 100644 index 2426ad0dba..0000000000 --- a/neuralnetworks/1.2/vts/functional/Models.h +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Copyright (C) 2018 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef VTS_HAL_NEURALNETWORKS_V1_2_VTS_FUNCTIONAL_MODELS_H -#define VTS_HAL_NEURALNETWORKS_V1_2_VTS_FUNCTIONAL_MODELS_H - -#define LOG_TAG "neuralnetworks_hidl_hal_test" - -#include "TestHarness.h" - -#include -#include -#include - -namespace android { -namespace hardware { -namespace neuralnetworks { -namespace V1_2 { -namespace vts { -namespace functional { - -using MixedTypedExample = test_helper::MixedTypedExample; - -#define FOR_EACH_TEST_MODEL(FN) FN(random_multinomial) - -#define FORWARD_DECLARE_GENERATED_OBJECTS(function) \ - namespace function { \ - extern std::vector examples; \ - Model createTestModel(); \ - } - -FOR_EACH_TEST_MODEL(FORWARD_DECLARE_GENERATED_OBJECTS) - -#undef FORWARD_DECLARE_GENERATED_OBJECTS - -} // namespace functional -} // namespace vts -} // namespace V1_2 -} // namespace neuralnetworks -} // namespace hardware -} // namespace android - -#endif // VTS_HAL_NEURALNETWORKS_V1_2_VTS_FUNCTIONAL_MODELS_H diff --git a/neuralnetworks/1.2/vts/functional/ModelsV1_0.h b/neuralnetworks/1.2/vts/functional/ModelsV1_0.h deleted file mode 100644 index e81e64bb41..0000000000 --- a/neuralnetworks/1.2/vts/functional/ModelsV1_0.h +++ /dev/null @@ -1,202 +0,0 @@ -/* - * Copyright (C) 2018 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef VTS_HAL_NEURALNETWORKS_V1_2_VTS_FUNCTIONAL_MODELS_V1_0_H -#define VTS_HAL_NEURALNETWORKS_V1_2_VTS_FUNCTIONAL_MODELS_V1_0_H - -#define LOG_TAG "neuralnetworks_hidl_hal_test" - -#include "TestHarness.h" - -#include -#include -#include - -namespace android { -namespace hardware { -namespace neuralnetworks { -namespace V1_2 { -namespace vts { -namespace functional { - -using MixedTypedExample = test_helper::MixedTypedExample; - -#define FOR_EACH_TEST_MODEL(FN) \ - FN(add_broadcast_quant8) \ - FN(add) \ - FN(add_quant8) \ - FN(avg_pool_float_1) \ - FN(avg_pool_float_2) \ - FN(avg_pool_float_3) \ - FN(avg_pool_float_4) \ - FN(avg_pool_float_5) \ - FN(avg_pool_quant8_1) \ - FN(avg_pool_quant8_2) \ - FN(avg_pool_quant8_3) \ - FN(avg_pool_quant8_4) \ - FN(avg_pool_quant8_5) \ - FN(concat_float_1) \ - FN(concat_float_2) \ - FN(concat_float_3) \ - FN(concat_quant8_1) \ - FN(concat_quant8_2) \ - FN(concat_quant8_3) \ - FN(conv_1_h3_w2_SAME) \ - FN(conv_1_h3_w2_VALID) \ - FN(conv_3_h3_w2_SAME) \ - FN(conv_3_h3_w2_VALID) \ - FN(conv_float_2) \ - FN(conv_float_channels) \ - FN(conv_float_channels_weights_as_inputs) \ - FN(conv_float_large) \ - FN(conv_float_large_weights_as_inputs) \ - FN(conv_float) \ - FN(conv_float_weights_as_inputs) \ - FN(conv_quant8_2) \ - FN(conv_quant8_channels) \ - FN(conv_quant8_channels_weights_as_inputs) \ - FN(conv_quant8_large) \ - FN(conv_quant8_large_weights_as_inputs) \ - FN(conv_quant8) \ - FN(conv_quant8_overflow) \ - FN(conv_quant8_overflow_weights_as_inputs) \ - FN(conv_quant8_weights_as_inputs) \ - FN(depth_to_space_float_1) \ - FN(depth_to_space_float_2) \ - FN(depth_to_space_float_3) \ - FN(depth_to_space_quant8_1) \ - FN(depth_to_space_quant8_2) \ - FN(depthwise_conv2d_float_2) \ - FN(depthwise_conv2d_float_large_2) \ - FN(depthwise_conv2d_float_large_2_weights_as_inputs) \ - FN(depthwise_conv2d_float_large) \ - FN(depthwise_conv2d_float_large_weights_as_inputs) \ - FN(depthwise_conv2d_float) \ - FN(depthwise_conv2d_float_weights_as_inputs) \ - FN(depthwise_conv2d_quant8_2) \ - FN(depthwise_conv2d_quant8_large) \ - FN(depthwise_conv2d_quant8_large_weights_as_inputs) \ - FN(depthwise_conv2d_quant8) \ - FN(depthwise_conv2d_quant8_weights_as_inputs) \ - FN(depthwise_conv) \ - FN(dequantize) \ - FN(embedding_lookup) \ - FN(floor) \ - FN(fully_connected_float_2) \ - FN(fully_connected_float_large) \ - FN(fully_connected_float_large_weights_as_inputs) \ - FN(fully_connected_float) \ - FN(fully_connected_float_weights_as_inputs) \ - FN(fully_connected_quant8_2) \ - FN(fully_connected_quant8_large) \ - FN(fully_connected_quant8_large_weights_as_inputs) \ - FN(fully_connected_quant8) \ - FN(fully_connected_quant8_weights_as_inputs) \ - FN(hashtable_lookup_float) \ - FN(hashtable_lookup_quant8) \ - FN(l2_normalization_2) \ - FN(l2_normalization_large) \ - FN(l2_normalization) \ - FN(l2_pool_float_2) \ - FN(l2_pool_float_large) \ - FN(l2_pool_float) \ - FN(local_response_norm_float_1) \ - FN(local_response_norm_float_2) \ - FN(local_response_norm_float_3) \ - FN(local_response_norm_float_4) \ - FN(logistic_float_1) \ - FN(logistic_float_2) \ - FN(logistic_quant8_1) \ - FN(logistic_quant8_2) \ - FN(lsh_projection_2) \ - FN(lsh_projection) \ - FN(lsh_projection_weights_as_inputs) \ - FN(lstm2) \ - FN(lstm2_state2) \ - FN(lstm2_state) \ - FN(lstm3) \ - FN(lstm3_state2) \ - FN(lstm3_state3) \ - FN(lstm3_state) \ - FN(lstm) \ - FN(lstm_state2) \ - FN(lstm_state) \ - FN(max_pool_float_1) \ - FN(max_pool_float_2) \ - FN(max_pool_float_3) \ - FN(max_pool_float_4) \ - FN(max_pool_quant8_1) \ - FN(max_pool_quant8_2) \ - FN(max_pool_quant8_3) \ - FN(max_pool_quant8_4) \ - FN(mobilenet_224_gender_basic_fixed) \ - FN(mobilenet_quantized) \ - FN(mul_broadcast_quant8) \ - FN(mul) \ - FN(mul_quant8) \ - FN(mul_relu) \ - FN(relu1_float_1) \ - FN(relu1_float_2) \ - FN(relu1_quant8_1) \ - FN(relu1_quant8_2) \ - FN(relu6_float_1) \ - FN(relu6_float_2) \ - FN(relu6_quant8_1) \ - FN(relu6_quant8_2) \ - FN(relu_float_1) \ - FN(relu_float_2) \ - FN(relu_quant8_1) \ - FN(relu_quant8_2) \ - FN(reshape) \ - FN(reshape_quant8) \ - FN(reshape_quant8_weights_as_inputs) \ - FN(reshape_weights_as_inputs) \ - FN(resize_bilinear_2) \ - FN(resize_bilinear) \ - FN(rnn) \ - FN(rnn_state) \ - FN(softmax_float_1) \ - FN(softmax_float_2) \ - FN(softmax_quant8_1) \ - FN(softmax_quant8_2) \ - FN(space_to_depth_float_1) \ - FN(space_to_depth_float_2) \ - FN(space_to_depth_float_3) \ - FN(space_to_depth_quant8_1) \ - FN(space_to_depth_quant8_2) \ - FN(svdf2) \ - FN(svdf) \ - FN(svdf_state) \ - FN(tanh) - -#define FORWARD_DECLARE_GENERATED_OBJECTS(function) \ - namespace function { \ - extern std::vector examples; \ - Model createTestModel(); \ - } - -FOR_EACH_TEST_MODEL(FORWARD_DECLARE_GENERATED_OBJECTS) - -#undef FORWARD_DECLARE_GENERATED_OBJECTS - -} // namespace functional -} // namespace vts -} // namespace V1_2 -} // namespace neuralnetworks -} // namespace hardware -} // namespace android - -#endif // VTS_HAL_NEURALNETWORKS_V1_2_VTS_FUNCTIONAL_MODELS_V1_0_H diff --git a/neuralnetworks/1.2/vts/functional/ModelsV1_1.h b/neuralnetworks/1.2/vts/functional/ModelsV1_1.h deleted file mode 100644 index eb68de4459..0000000000 --- a/neuralnetworks/1.2/vts/functional/ModelsV1_1.h +++ /dev/null @@ -1,230 +0,0 @@ -/* - * Copyright (C) 2018 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef VTS_HAL_NEURALNETWORKS_V1_1_VTS_FUNCTIONAL_MODELS_H -#define VTS_HAL_NEURALNETWORKS_V1_1_VTS_FUNCTIONAL_MODELS_H - -#define LOG_TAG "neuralnetworks_hidl_hal_test" - -#include "TestHarness.h" - -#include - -namespace android { -namespace hardware { -namespace neuralnetworks { -namespace V1_2 { -namespace vts { -namespace functional { - -using MixedTypedExample = test_helper::MixedTypedExample; - -#define FOR_EACH_TEST_MODEL(FN) \ - FN(add_relaxed) \ - FN(avg_pool_float_1_relaxed) \ - FN(avg_pool_float_2_relaxed) \ - FN(avg_pool_float_3_relaxed) \ - FN(avg_pool_float_4_relaxed) \ - FN(avg_pool_float_5_relaxed) \ - FN(batch_to_space) \ - FN(batch_to_space_float_1) \ - FN(batch_to_space_float_1_relaxed) \ - FN(batch_to_space_quant8_1) \ - FN(batch_to_space_relaxed) \ - FN(concat_float_1_relaxed) \ - FN(concat_float_2_relaxed) \ - FN(concat_float_3_relaxed) \ - FN(conv_1_h3_w2_SAME_relaxed) \ - FN(conv_1_h3_w2_VALID_relaxed) \ - FN(conv_3_h3_w2_SAME_relaxed) \ - FN(conv_3_h3_w2_VALID_relaxed) \ - FN(conv_float_2_relaxed) \ - FN(conv_float_channels_relaxed) \ - FN(conv_float_channels_weights_as_inputs_relaxed) \ - FN(conv_float_large_relaxed) \ - FN(conv_float_large_weights_as_inputs_relaxed) \ - FN(conv_float_relaxed) \ - FN(conv_float_weights_as_inputs_relaxed) \ - FN(depth_to_space_float_1_relaxed) \ - FN(depth_to_space_float_2_relaxed) \ - FN(depth_to_space_float_3_relaxed) \ - FN(depthwise_conv2d_float_2_relaxed) \ - FN(depthwise_conv2d_float_large_2_relaxed) \ - FN(depthwise_conv2d_float_large_2_weights_as_inputs_relaxed) \ - FN(depthwise_conv2d_float_large_relaxed) \ - FN(depthwise_conv2d_float_large_weights_as_inputs_relaxed) \ - FN(depthwise_conv2d_float_relaxed) \ - FN(depthwise_conv2d_float_weights_as_inputs_relaxed) \ - FN(depthwise_conv_relaxed) \ - FN(dequantize_relaxed) \ - FN(div) \ - FN(div_broadcast_float) \ - FN(div_broadcast_float_relaxed) \ - FN(div_relaxed) \ - FN(embedding_lookup_relaxed) \ - FN(floor_relaxed) \ - FN(fully_connected_float_2_relaxed) \ - FN(fully_connected_float_4d_simple) \ - FN(fully_connected_float_4d_simple_relaxed) \ - FN(fully_connected_float_large_relaxed) \ - FN(fully_connected_float_large_weights_as_inputs_relaxed) \ - FN(fully_connected_float_relaxed) \ - FN(fully_connected_float_weights_as_inputs_relaxed) \ - FN(hashtable_lookup_float_relaxed) \ - FN(l2_normalization_2_relaxed) \ - FN(l2_normalization_large_relaxed) \ - FN(l2_normalization_relaxed) \ - FN(l2_pool_float_2_relaxed) \ - FN(l2_pool_float_large_relaxed) \ - FN(l2_pool_float_relaxed) \ - FN(local_response_norm_float_1_relaxed) \ - FN(local_response_norm_float_2_relaxed) \ - FN(local_response_norm_float_3_relaxed) \ - FN(local_response_norm_float_4_relaxed) \ - FN(logistic_float_1_relaxed) \ - FN(logistic_float_2_relaxed) \ - FN(lsh_projection_2_relaxed) \ - FN(lsh_projection_relaxed) \ - FN(lsh_projection_weights_as_inputs_relaxed) \ - FN(lstm2_relaxed) \ - FN(lstm2_state2_relaxed) \ - FN(lstm2_state_relaxed) \ - FN(lstm3_relaxed) \ - FN(lstm3_state2_relaxed) \ - FN(lstm3_state3_relaxed) \ - FN(lstm3_state_relaxed) \ - FN(lstm_relaxed) \ - FN(lstm_state2_relaxed) \ - FN(lstm_state_relaxed) \ - FN(max_pool_float_1_relaxed) \ - FN(max_pool_float_2_relaxed) \ - FN(max_pool_float_3_relaxed) \ - FN(max_pool_float_4_relaxed) \ - FN(mean) \ - FN(mean_float_1) \ - FN(mean_float_1_relaxed) \ - FN(mean_float_2) \ - FN(mean_float_2_relaxed) \ - FN(mean_quant8_1) \ - FN(mean_quant8_2) \ - FN(mean_relaxed) \ - FN(mobilenet_224_gender_basic_fixed_relaxed) \ - FN(mul_relaxed) \ - FN(mul_relu_relaxed) \ - FN(pad) \ - FN(pad_float_1) \ - FN(pad_float_1_relaxed) \ - FN(pad_relaxed) \ - FN(relu1_float_1_relaxed) \ - FN(relu1_float_2_relaxed) \ - FN(relu6_float_1_relaxed) \ - FN(relu6_float_2_relaxed) \ - FN(relu_float_1_relaxed) \ - FN(relu_float_2_relaxed) \ - FN(reshape_relaxed) \ - FN(reshape_weights_as_inputs_relaxed) \ - FN(resize_bilinear_2_relaxed) \ - FN(resize_bilinear_relaxed) \ - FN(rnn_relaxed) \ - FN(rnn_state_relaxed) \ - FN(softmax_float_1_relaxed) \ - FN(softmax_float_2_relaxed) \ - FN(space_to_batch) \ - FN(space_to_batch_float_1) \ - FN(space_to_batch_float_1_relaxed) \ - FN(space_to_batch_float_2) \ - FN(space_to_batch_float_2_relaxed) \ - FN(space_to_batch_float_3) \ - FN(space_to_batch_float_3_relaxed) \ - FN(space_to_batch_quant8_1) \ - FN(space_to_batch_quant8_2) \ - FN(space_to_batch_quant8_3) \ - FN(space_to_batch_relaxed) \ - FN(space_to_depth_float_1_relaxed) \ - FN(space_to_depth_float_2_relaxed) \ - FN(space_to_depth_float_3_relaxed) \ - FN(squeeze) \ - FN(squeeze_float_1) \ - FN(squeeze_float_1_relaxed) \ - FN(squeeze_quant8_1) \ - FN(squeeze_relaxed) \ - FN(strided_slice) \ - FN(strided_slice_float_1) \ - FN(strided_slice_float_10) \ - FN(strided_slice_float_10_relaxed) \ - FN(strided_slice_float_11) \ - FN(strided_slice_float_11_relaxed) \ - FN(strided_slice_float_1_relaxed) \ - FN(strided_slice_float_2) \ - FN(strided_slice_float_2_relaxed) \ - FN(strided_slice_float_3) \ - FN(strided_slice_float_3_relaxed) \ - FN(strided_slice_float_4) \ - FN(strided_slice_float_4_relaxed) \ - FN(strided_slice_float_5) \ - FN(strided_slice_float_5_relaxed) \ - FN(strided_slice_float_6) \ - FN(strided_slice_float_6_relaxed) \ - FN(strided_slice_float_7) \ - FN(strided_slice_float_7_relaxed) \ - FN(strided_slice_float_8) \ - FN(strided_slice_float_8_relaxed) \ - FN(strided_slice_float_9) \ - FN(strided_slice_float_9_relaxed) \ - FN(strided_slice_qaunt8_10) \ - FN(strided_slice_qaunt8_11) \ - FN(strided_slice_quant8_1) \ - FN(strided_slice_quant8_2) \ - FN(strided_slice_quant8_3) \ - FN(strided_slice_quant8_4) \ - FN(strided_slice_quant8_5) \ - FN(strided_slice_quant8_6) \ - FN(strided_slice_quant8_7) \ - FN(strided_slice_quant8_8) \ - FN(strided_slice_quant8_9) \ - FN(strided_slice_relaxed) \ - FN(sub) \ - FN(sub_broadcast_float) \ - FN(sub_broadcast_float_relaxed) \ - FN(sub_relaxed) \ - FN(svdf2_relaxed) \ - FN(svdf_relaxed) \ - FN(svdf_state_relaxed) \ - FN(tanh_relaxed) \ - FN(transpose) \ - FN(transpose_float_1) \ - FN(transpose_float_1_relaxed) \ - FN(transpose_quant8_1) \ - FN(transpose_relaxed) - -#define FORWARD_DECLARE_GENERATED_OBJECTS(function) \ - namespace function { \ - extern std::vector examples; \ - Model createTestModel(); \ - } - -FOR_EACH_TEST_MODEL(FORWARD_DECLARE_GENERATED_OBJECTS) - -#undef FORWARD_DECLARE_GENERATED_OBJECTS - -} // namespace functional -} // namespace vts -} // namespace V1_2 -} // namespace neuralnetworks -} // namespace hardware -} // namespace android - -#endif // VTS_HAL_NEURALNETWORKS_V1_2_VTS_FUNCTIONAL_MODELS_V1_1_H diff --git a/neuralnetworks/1.2/vts/functional/ValidationTests.cpp b/neuralnetworks/1.2/vts/functional/ValidationTests.cpp deleted file mode 100644 index 3bdc5cdea7..0000000000 --- a/neuralnetworks/1.2/vts/functional/ValidationTests.cpp +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright (C) 2018 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#define LOG_TAG "neuralnetworks_hidl_hal_test" - -#include "Models.h" -#include "VtsHalNeuralnetworks.h" - -namespace android { -namespace hardware { -namespace neuralnetworks { -namespace V1_2 { -namespace vts { -namespace functional { - -// forward declarations -std::vector createRequests(const std::vector& examples); - -// generate validation tests -#define VTS_CURRENT_TEST_CASE(TestName) \ - TEST_F(ValidationTest, TestName) { \ - const Model model = TestName::createTestModel(); \ - const std::vector requests = createRequests(TestName::examples); \ - validateModel(model); \ - validateRequests(model, requests); \ - } - -FOR_EACH_TEST_MODEL(VTS_CURRENT_TEST_CASE) - -#undef VTS_CURRENT_TEST_CASE - -} // namespace functional -} // namespace vts -} // namespace V1_2 -} // namespace neuralnetworks -} // namespace hardware -} // namespace android diff --git a/neuralnetworks/1.2/vts/functional/ValidationTestsV1_0.cpp b/neuralnetworks/1.2/vts/functional/ValidationTestsV1_0.cpp deleted file mode 100644 index c54ed4370a..0000000000 --- a/neuralnetworks/1.2/vts/functional/ValidationTestsV1_0.cpp +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright (C) 2018 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#define LOG_TAG "neuralnetworks_hidl_hal_test" - -#include "ModelsV1_0.h" -#include "VtsHalNeuralnetworks.h" - -namespace android { -namespace hardware { -namespace neuralnetworks { -namespace V1_2 { -namespace vts { -namespace functional { - -// forward declarations -std::vector createRequests(const std::vector& examples); - -// generate validation tests -#define VTS_CURRENT_TEST_CASE(TestName) \ - TEST_F(ValidationTest, TestName) { \ - const Model model = TestName::createTestModel(); \ - const std::vector requests = createRequests(TestName::examples); \ - validateModel(model); \ - validateRequests(model, requests); \ - } - -FOR_EACH_TEST_MODEL(VTS_CURRENT_TEST_CASE) - -#undef VTS_CURRENT_TEST_CASE - -} // namespace functional -} // namespace vts -} // namespace V1_2 -} // namespace neuralnetworks -} // namespace hardware -} // namespace android diff --git a/neuralnetworks/1.2/vts/functional/ValidationTestsV1_1.cpp b/neuralnetworks/1.2/vts/functional/ValidationTestsV1_1.cpp deleted file mode 100644 index 95932d59c9..0000000000 --- a/neuralnetworks/1.2/vts/functional/ValidationTestsV1_1.cpp +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright (C) 2018 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#define LOG_TAG "neuralnetworks_hidl_hal_test" - -#include "ModelsV1_1.h" -#include "VtsHalNeuralnetworks.h" - -namespace android { -namespace hardware { -namespace neuralnetworks { -namespace V1_2 { -namespace vts { -namespace functional { - -// forward declarations -std::vector createRequests(const std::vector& examples); - -// generate validation tests -#define VTS_CURRENT_TEST_CASE(TestName) \ - TEST_F(ValidationTest, TestName) { \ - const Model model = TestName::createTestModel(); \ - const std::vector requests = createRequests(TestName::examples); \ - validateModel(model); \ - validateRequests(model, requests); \ - } - -FOR_EACH_TEST_MODEL(VTS_CURRENT_TEST_CASE) - -#undef VTS_CURRENT_TEST_CASE - -} // namespace functional -} // namespace vts -} // namespace V1_2 -} // namespace neuralnetworks -} // namespace hardware -} // namespace android From 650fd186e4faf057807bc8c19e7d1d97cca6b42e Mon Sep 17 00:00:00 2001 From: "Michael K. Sanders" Date: Tue, 30 Oct 2018 14:44:48 +0000 Subject: [PATCH 4/6] Adds float16 support to generated tests. Uses the _Float16 type (ISO/IEC TS 18661-3:2015) for storage. Test: VtsHalNeuralnetworksV1_2TargetTest --hal_service_instance=android.hardware.neuralnetworks@1.2::IDevice/sample-all Bug: 113563458 Change-Id: I1779b828d397b5354dc854c68d21c159cd5b582c Merged-In: I1779b828d397b5354dc854c68d21c159cd5b582c (cherry picked from commit efa4c814fb463a07d561393e4ba78a9a47e266e4) --- .../vts/functional/GeneratedTestHarness.cpp | 39 ++++++++++--------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/neuralnetworks/1.0/vts/functional/GeneratedTestHarness.cpp b/neuralnetworks/1.0/vts/functional/GeneratedTestHarness.cpp index 1f66c43bf9..802d01875c 100644 --- a/neuralnetworks/1.0/vts/functional/GeneratedTestHarness.cpp +++ b/neuralnetworks/1.0/vts/functional/GeneratedTestHarness.cpp @@ -45,6 +45,7 @@ using ::test_helper::for_each; using ::test_helper::Int32Operands; using ::test_helper::MixedTyped; using ::test_helper::MixedTypedExample; +using ::test_helper::MixedTypedIndex; using ::test_helper::Quant8Operands; using ::test_helper::resize_accordingly; @@ -63,14 +64,16 @@ void copy_back(MixedTyped* dst, const std::vector& ra, char* sr copy_back_(dst, ra, src); copy_back_(dst, ra, src); copy_back_(dst, ra, src); - static_assert(4 == std::tuple_size::value, + copy_back_<_Float16>(dst, ra, src); + static_assert(5 == std::tuple_size::value, "Number of types in MixedTyped changed, but copy_back function wasn't updated"); } // Top level driver for models and examples generated by test_generator.py // Test driver for those generated from ml/nn/runtime/test/spec void EvaluatePreparedModel(sp& preparedModel, std::function is_ignored, - const std::vector& examples, float fpAtol = 1e-5f, + const std::vector& examples, + bool hasRelaxedFloat32Model = false, float fpAtol = 1e-5f, float fpRtol = 1e-5f) { const uint32_t INPUT = 0; const uint32_t OUTPUT = 1; @@ -78,13 +81,20 @@ void EvaluatePreparedModel(sp& preparedModel, std::function::index>(inputs).empty(); + if (hasRelaxedFloat32Model || hasFloat16Inputs) { + // TODO: Adjust the error limit based on testing. + // If in relaxed mode, set the absolute tolerance to be 5ULP of FP16. + fpAtol = 5.0f * 0.0009765625f; + // Set the relative tolerance to be 5ULP of the corresponding FP precision. + fpRtol = 5.0f * 0.0009765625f; + } + std::vector inputs_info, outputs_info; uint32_t inputSize = 0, outputSize = 0; - // This function only partially specifies the metadata (vector of RequestArguments). // The contents are copied over below. for_all(inputs, [&inputs_info, &inputSize](int index, auto, auto s) { @@ -228,7 +238,8 @@ void Execute(const sp& device, std::function c ASSERT_NE(nullptr, preparedModel.get()); float fpAtol = 1e-5f, fpRtol = 5.0f * 1.1920928955078125e-7f; - EvaluatePreparedModel(preparedModel, is_ignored, examples, fpAtol, fpRtol); + EvaluatePreparedModel(preparedModel, is_ignored, examples, + /*hasRelaxedFloat32Model=*/false, fpAtol, fpRtol); } void Execute(const sp& device, std::function create_model, @@ -272,13 +283,8 @@ void Execute(const sp& device, std::function c EXPECT_EQ(ErrorStatus::NONE, prepareReturnStatus); ASSERT_NE(nullptr, preparedModel.get()); - // TODO: Adjust the error limit based on testing. - // If in relaxed mode, set the absolute tolerance to be 5ULP of FP16. - float fpAtol = !model.relaxComputationFloat32toFloat16 ? 1e-5f : 5.0f * 0.0009765625f; - // Set the relative tolerance to be 5ULP of the corresponding FP precision. - float fpRtol = !model.relaxComputationFloat32toFloat16 ? 5.0f * 1.1920928955078125e-7f - : 5.0f * 0.0009765625f; - EvaluatePreparedModel(preparedModel, is_ignored, examples, fpAtol, fpRtol); + EvaluatePreparedModel(preparedModel, is_ignored, examples, + model.relaxComputationFloat32toFloat16); } // TODO: Reduce code duplication. @@ -323,13 +329,8 @@ void Execute(const sp& device, std::function c EXPECT_EQ(ErrorStatus::NONE, prepareReturnStatus); ASSERT_NE(nullptr, preparedModel.get()); - // TODO: Adjust the error limit based on testing. - // If in relaxed mode, set the absolute tolerance to be 5ULP of FP16. - float fpAtol = !model.relaxComputationFloat32toFloat16 ? 1e-5f : 5.0f * 0.0009765625f; - // Set the relative tolerance to be 5ULP of the corresponding FP precision. - float fpRtol = !model.relaxComputationFloat32toFloat16 ? 5.0f * 1.1920928955078125e-7f - : 5.0f * 0.0009765625f; - EvaluatePreparedModel(preparedModel, is_ignored, examples, fpAtol, fpRtol); + EvaluatePreparedModel(preparedModel, is_ignored, examples, + model.relaxComputationFloat32toFloat16); } } // namespace generated_tests From 1a2492f31141acf8690d36ea2b1d476dcc26932a Mon Sep 17 00:00:00 2001 From: Xusong Wang Date: Fri, 5 Oct 2018 11:49:13 -0700 Subject: [PATCH 5/6] Fix VTS ValidationTest for 1.2 ops. Fix VTS ValidationTest for 1.2 ops by skipping invalid tests. Bug: 119425787 Test: VtsHalNeuralnetworksV1_xTargetTest with 1.2 sample driver Change-Id: I61e33d8ff3d9121d9ceb28e667ba6c00bb9951ac Merged-In: I61e33d8ff3d9121d9ceb28e667ba6c00bb9951ac (cherry picked from commit 5b747ae02b9c94df4b3d95565c37a343c2d200c0) --- .../1.2/vts/functional/ValidateModel.cpp | 145 ++++++++++++++---- 1 file changed, 119 insertions(+), 26 deletions(-) diff --git a/neuralnetworks/1.2/vts/functional/ValidateModel.cpp b/neuralnetworks/1.2/vts/functional/ValidateModel.cpp index b4921e3305..e3096426da 100644 --- a/neuralnetworks/1.2/vts/functional/ValidateModel.cpp +++ b/neuralnetworks/1.2/vts/functional/ValidateModel.cpp @@ -224,8 +224,9 @@ static std::vector getInvalidZeroPoints(OperandType type) { case OperandType::TENSOR_INT32: return {1}; case OperandType::TENSOR_QUANT8_ASYMM: - case OperandType::TENSOR_QUANT16_ASYMM: return {-1, 256}; + case OperandType::TENSOR_QUANT16_ASYMM: + return {-32769, 32768}; default: return {}; } @@ -291,15 +292,33 @@ static void mutateOperand(Operand* operand, OperandType type) { *operand = newOperand; } -static bool mutateOperationOperandTypeSkip(size_t operand, const Model& model) { - // LSH_PROJECTION's second argument is allowed to have any type. This is the - // only operation that currently has a type that can be anything independent - // from any other type. Changing the operand type to any other type will - // result in a valid model for LSH_PROJECTION. If this is the case, skip the - // test. +static bool mutateOperationOperandTypeSkip(size_t operand, OperandType type, const Model& model) { + // Do not test OEM types + if (type == model.operands[operand].type || type == OperandType::OEM || + type == OperandType::TENSOR_OEM_BYTE) { + return true; + } for (const Operation& operation : model.operations) { - if (operation.type == OperationType::LSH_PROJECTION && operand == operation.inputs[1]) { - return true; + // Skip mutateOperationOperandTypeTest for the following operations. + // - LSH_PROJECTION's second argument is allowed to have any type. + // - ARGMIN and ARGMAX's first argument can be any of TENSOR_(FLOAT32|INT32|QUANT8_ASYMM). + // - CAST's argument can be any of TENSOR_(FLOAT32|INT32|QUANT8_ASYMM). + switch (operation.type) { + case OperationType::LSH_PROJECTION: { + if (operand == operation.inputs[1]) { + return true; + } + } break; + case OperationType::CAST: + case OperationType::ARGMAX: + case OperationType::ARGMIN: { + if (type == OperandType::TENSOR_FLOAT32 || type == OperandType::TENSOR_INT32 || + type == OperandType::TENSOR_QUANT8_ASYMM) { + return true; + } + } break; + default: + break; } } return false; @@ -307,14 +326,8 @@ static bool mutateOperationOperandTypeSkip(size_t operand, const Model& model) { static void mutateOperationOperandTypeTest(const sp& device, const Model& model) { for (size_t operand = 0; operand < model.operands.size(); ++operand) { - if (mutateOperationOperandTypeSkip(operand, model)) { - continue; - } for (OperandType invalidOperandType : hidl_enum_range{}) { - // Do not test OEM types - if (invalidOperandType == model.operands[operand].type || - invalidOperandType == OperandType::OEM || - invalidOperandType == OperandType::TENSOR_OEM_BYTE) { + if (mutateOperationOperandTypeSkip(operand, invalidOperandType, model)) { continue; } const std::string message = "mutateOperationOperandTypeTest: operand " + @@ -406,8 +419,26 @@ static void removeOperand(Model* model, uint32_t index) { removeValueAndDecrementGreaterValues(&model->outputIndexes, index); } +static bool removeOperandSkip(size_t operand, const Model& model) { + for (const Operation& operation : model.operations) { + // Skip removeOperandTest for the following operations. + // - SPLIT's outputs are not checked during prepareModel. + if (operation.type == OperationType::SPLIT) { + for (const size_t outOprand : operation.outputs) { + if (operand == outOprand) { + return true; + } + } + } + } + return false; +} + static void removeOperandTest(const sp& device, const Model& model) { for (size_t operand = 0; operand < model.operands.size(); ++operand) { + if (removeOperandSkip(operand, model)) { + continue; + } const std::string message = "removeOperandTest: operand " + std::to_string(operand); validate(device, message, model, [operand](Model* model) { removeOperand(model, operand); }); @@ -433,15 +464,76 @@ static void removeOperationTest(const sp& device, const Model& model) { ///////////////////////// REMOVE OPERATION INPUT ///////////////////////// +static bool removeOperationInputSkip(const Operation& op, size_t input) { + // Skip removeOperationInputTest for the following operations. + // - CONCATENATION has at least 2 inputs, with the last element being INT32. + // - CONV_2D, DEPTHWISE_CONV_2D, MAX_POOL_2D, AVERAGE_POOL_2D, L2_POOL_2D, RESIZE_BILINEAR, + // SPACE_TO_DEPTH, SPACE_TO_DEPTH, SPACE_TO_BATCH_ND, BATCH_TO_SPACE_ND can have an optional + // layout parameter. + // - L2_NORMALIZATION, LOCAL_RESPONSE_NORMALIZATION, SOFTMAX can have an optional axis + // parameter. + switch (op.type) { + case OperationType::CONCATENATION: { + if (op.inputs.size() > 2 && input != op.inputs.size() - 1) { + return true; + } + } break; + case OperationType::DEPTHWISE_CONV_2D: { + if ((op.inputs.size() == 12 && input == 11) || (op.inputs.size() == 9 && input == 8)) { + return true; + } + } break; + case OperationType::CONV_2D: + case OperationType::AVERAGE_POOL_2D: + case OperationType::MAX_POOL_2D: + case OperationType::L2_POOL_2D: { + if ((op.inputs.size() == 11 && input == 10) || (op.inputs.size() == 8 && input == 7)) { + return true; + } + } break; + case OperationType::RESIZE_BILINEAR: { + if (op.inputs.size() == 4 && input == 3) { + return true; + } + } break; + case OperationType::SPACE_TO_DEPTH: + case OperationType::DEPTH_TO_SPACE: + case OperationType::BATCH_TO_SPACE_ND: { + if (op.inputs.size() == 3 && input == 2) { + return true; + } + } break; + case OperationType::SPACE_TO_BATCH_ND: { + if (op.inputs.size() == 4 && input == 3) { + return true; + } + } break; + case OperationType::L2_NORMALIZATION: { + if (op.inputs.size() == 2 && input == 1) { + return true; + } + } break; + case OperationType::LOCAL_RESPONSE_NORMALIZATION: { + if (op.inputs.size() == 6 && input == 5) { + return true; + } + } break; + case OperationType::SOFTMAX: { + if (op.inputs.size() == 3 && input == 2) { + return true; + } + } break; + default: + break; + } + return false; +} + static void removeOperationInputTest(const sp& device, const Model& model) { for (size_t operation = 0; operation < model.operations.size(); ++operation) { for (size_t input = 0; input < model.operations[operation].inputs.size(); ++input) { const Operation& op = model.operations[operation]; - // CONCATENATION has at least 2 inputs, with the last element being - // INT32. Skip this test if removing one of CONCATENATION's - // inputs still produces a valid model. - if (op.type == OperationType::CONCATENATION && op.inputs.size() > 2 && - input != op.inputs.size() - 1) { + if (removeOperationInputSkip(op, input)) { continue; } const std::string message = "removeOperationInputTest: operation " + @@ -479,12 +571,13 @@ static void removeOperationOutputTest(const sp& device, const Model& mo ///////////////////////// ADD OPERATION INPUT ///////////////////////// -static bool addOperationInputSkip(const Operation& operation) { +static bool addOperationInputSkip(const Operation& op) { // Skip addOperationInputTest for the following operations. - // L2_NORMALIZATION, LOCAL_RESPONSE_NORMALIZATION, SOFTMAX can have an optional axis parameter. - if (operation.type == OperationType::L2_NORMALIZATION || - operation.type == OperationType::LOCAL_RESPONSE_NORMALIZATION || - operation.type == OperationType::SOFTMAX) { + // - L2_NORMALIZATION, LOCAL_RESPONSE_NORMALIZATION, SOFTMAX can have an optional INT32 axis + // parameter. + if ((op.type == OperationType::L2_NORMALIZATION && op.inputs.size() == 1) || + (op.type == OperationType::LOCAL_RESPONSE_NORMALIZATION && op.inputs.size() == 5) || + (op.type == OperationType::SOFTMAX && op.inputs.size() == 2)) { return true; } return false; From 217c4071cc00d4e18171ccdb992ee37d006eb48d Mon Sep 17 00:00:00 2001 From: Lev Proleev Date: Tue, 13 Nov 2018 15:42:36 +0000 Subject: [PATCH 6/6] Replace TENSOR_QUANT16_ASYMM with TENSOR_QUANT16_SYMM * Update doc string * Update zero point mutation to check for symmetric quantization Fix: 118671831 Test: VtsHalNeuralnetworksV1_2TargetTest Change-Id: Id1999c793c839b892cfe45cbb245611b12db2a72 Merged-In: Id1999c793c839b892cfe45cbb245611b12db2a72 (cherry picked from commit 48c8820bac243e7eaff680056f2145f1b3b1ab2a) --- neuralnetworks/1.2/types.hal | 12 +++++------- neuralnetworks/1.2/vts/functional/ValidateModel.cpp | 10 +++++----- 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/neuralnetworks/1.2/types.hal b/neuralnetworks/1.2/types.hal index 366e626578..fe9b312c2b 100644 --- a/neuralnetworks/1.2/types.hal +++ b/neuralnetworks/1.2/types.hal @@ -33,15 +33,13 @@ enum OperandType : @1.0::OperandType { /** * A tensor of 16 bit signed integers that represent real numbers. * - * Attached to this tensor are two numbers that are used to convert the 16 - * bit integer to the real value and vice versa. These two numbers are: - * - scale: a 32 bit floating point value greater than zero. - * - zeroPoint: a 32 bit integer, in range [-32768, 32767]. + * Attached to this tensor is a number representing real value scale that is + * used to convert the 16 bit number to a real value in the following way: + * realValue = integerValue * scale. * - * The formula is: - * realValue = (integerValue - zeroPoint) * scale. + * scale is a 32 bit floating point with value greater then zero. */ - TENSOR_QUANT16_ASYMM = 7, + TENSOR_QUANT16_SYMM = 7, /** A tensor of 16 bit floating point values. */ TENSOR_FLOAT16 = 8, }; diff --git a/neuralnetworks/1.2/vts/functional/ValidateModel.cpp b/neuralnetworks/1.2/vts/functional/ValidateModel.cpp index e3096426da..c4f1b5e724 100644 --- a/neuralnetworks/1.2/vts/functional/ValidateModel.cpp +++ b/neuralnetworks/1.2/vts/functional/ValidateModel.cpp @@ -161,7 +161,7 @@ static uint32_t getInvalidRank(OperandType type) { case OperandType::TENSOR_FLOAT32: case OperandType::TENSOR_INT32: case OperandType::TENSOR_QUANT8_ASYMM: - case OperandType::TENSOR_QUANT16_ASYMM: + case OperandType::TENSOR_QUANT16_SYMM: return 0; default: return 0; @@ -193,7 +193,7 @@ static float getInvalidScale(OperandType type) { case OperandType::TENSOR_INT32: return -1.0f; case OperandType::TENSOR_QUANT8_ASYMM: - case OperandType::TENSOR_QUANT16_ASYMM: + case OperandType::TENSOR_QUANT16_SYMM: return 0.0f; default: return 0.0f; @@ -225,8 +225,8 @@ static std::vector getInvalidZeroPoints(OperandType type) { return {1}; case OperandType::TENSOR_QUANT8_ASYMM: return {-1, 256}; - case OperandType::TENSOR_QUANT16_ASYMM: - return {-32769, 32768}; + case OperandType::TENSOR_QUANT16_SYMM: + return {-32769, -1, 1, 32768}; default: return {}; } @@ -279,7 +279,7 @@ static void mutateOperand(Operand* operand, OperandType type) { newOperand.zeroPoint = 0; break; case OperandType::TENSOR_QUANT8_ASYMM: - case OperandType::TENSOR_QUANT16_ASYMM: + case OperandType::TENSOR_QUANT16_SYMM: newOperand.dimensions = operand->dimensions.size() > 0 ? operand->dimensions : hidl_vec({1}); newOperand.scale = operand->scale != 0.0f ? operand->scale : 1.0f;