diff --git a/neuralnetworks/1.3/vts/OWNERS b/neuralnetworks/1.3/vts/OWNERS deleted file mode 100644 index b5a8e1f473..0000000000 --- a/neuralnetworks/1.3/vts/OWNERS +++ /dev/null @@ -1,16 +0,0 @@ -# Neuralnetworks team -butlermichael@google.com -dgross@google.com -jeanluc@google.com -levp@google.com -miaowang@google.com -mikie@google.com -mks@google.com -pszczepaniak@google.com -slavash@google.com -vddang@google.com -xusongw@google.com - -# VTS team -yim@google.com -yuexima@google.com diff --git a/neuralnetworks/1.3/vts/functional/BasicTests.cpp b/neuralnetworks/1.3/vts/functional/BasicTests.cpp deleted file mode 100644 index 8e82c5376e..0000000000 --- a/neuralnetworks/1.3/vts/functional/BasicTests.cpp +++ /dev/null @@ -1,114 +0,0 @@ -/* - * Copyright (C) 2018 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#define LOG_TAG "neuralnetworks_hidl_hal_test" - -#include "VtsHalNeuralnetworks.h" - -namespace android::hardware::neuralnetworks::V1_2::vts::functional { - -using V1_0::DeviceStatus; -using V1_0::ErrorStatus; -using V1_0::PerformanceInfo; - -// create device test -TEST_P(NeuralnetworksHidlTest, CreateDevice) {} - -// status test -TEST_P(NeuralnetworksHidlTest, StatusTest) { - Return status = kDevice->getStatus(); - ASSERT_TRUE(status.isOk()); - EXPECT_EQ(DeviceStatus::AVAILABLE, static_cast(status)); -} - -// initialization -TEST_P(NeuralnetworksHidlTest, GetCapabilitiesTest) { - using OperandPerformance = Capabilities::OperandPerformance; - Return ret = kDevice->getCapabilities_1_2([](ErrorStatus status, - const Capabilities& capabilities) { - EXPECT_EQ(ErrorStatus::NONE, status); - - auto isPositive = [](const PerformanceInfo& perf) { - return perf.execTime > 0.0f && perf.powerUsage > 0.0f; - }; - - EXPECT_TRUE(isPositive(capabilities.relaxedFloat32toFloat16PerformanceScalar)); - EXPECT_TRUE(isPositive(capabilities.relaxedFloat32toFloat16PerformanceTensor)); - const auto& opPerf = capabilities.operandPerformance; - EXPECT_TRUE(std::all_of( - opPerf.begin(), opPerf.end(), - [isPositive](const OperandPerformance& a) { return isPositive(a.info); })); - EXPECT_TRUE(std::is_sorted(opPerf.begin(), opPerf.end(), - [](const OperandPerformance& a, const OperandPerformance& b) { - return a.type < b.type; - })); - }); - EXPECT_TRUE(ret.isOk()); -} - -// device version test -TEST_P(NeuralnetworksHidlTest, GetDeviceVersionStringTest) { - Return ret = - kDevice->getVersionString([](ErrorStatus status, const hidl_string& version) { - EXPECT_EQ(ErrorStatus::NONE, status); - EXPECT_LT(0, version.size()); - }); - EXPECT_TRUE(ret.isOk()); -} - -// device type test -TEST_P(NeuralnetworksHidlTest, GetDeviceTypeTest) { - Return ret = kDevice->getType([](ErrorStatus status, DeviceType type) { - EXPECT_EQ(ErrorStatus::NONE, status); - EXPECT_TRUE(type == DeviceType::OTHER || type == DeviceType::CPU || - type == DeviceType::GPU || type == DeviceType::ACCELERATOR); - }); - EXPECT_TRUE(ret.isOk()); -} - -// device supported extensions test -TEST_P(NeuralnetworksHidlTest, GetDeviceSupportedExtensionsTest) { - Return ret = kDevice->getSupportedExtensions( - [](ErrorStatus status, const hidl_vec& extensions) { - EXPECT_EQ(ErrorStatus::NONE, status); - for (auto& extension : extensions) { - std::string extensionName = extension.name; - EXPECT_FALSE(extensionName.empty()); - for (char c : extensionName) { - EXPECT_TRUE(('a' <= c && c <= 'z') || ('0' <= c && c <= '9') || c == '_' || - c == '.') - << "Extension name contains an illegal character: " << c; - } - EXPECT_NE(extensionName.find('.'), std::string::npos) - << "Extension name must start with the reverse domain name of the " - "vendor"; - } - }); - EXPECT_TRUE(ret.isOk()); -} - -// getNumberOfCacheFilesNeeded test -TEST_P(NeuralnetworksHidlTest, getNumberOfCacheFilesNeeded) { - Return ret = kDevice->getNumberOfCacheFilesNeeded( - [](ErrorStatus status, uint32_t numModelCache, uint32_t numDataCache) { - EXPECT_EQ(ErrorStatus::NONE, status); - EXPECT_LE(numModelCache, - static_cast(Constant::MAX_NUMBER_OF_CACHE_FILES)); - EXPECT_LE(numDataCache, static_cast(Constant::MAX_NUMBER_OF_CACHE_FILES)); - }); - EXPECT_TRUE(ret.isOk()); -} -} // namespace android::hardware::neuralnetworks::V1_2::vts::functional diff --git a/neuralnetworks/1.3/vts/functional/Callbacks.cpp b/neuralnetworks/1.3/vts/functional/Callbacks.cpp deleted file mode 100644 index 3972ad6ff2..0000000000 --- a/neuralnetworks/1.3/vts/functional/Callbacks.cpp +++ /dev/null @@ -1,143 +0,0 @@ -/* - * Copyright (C) 2019 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#define LOG_TAG "Callbacks" - -#include "1.2/Callbacks.h" - -#include - -#include - -namespace android::hardware::neuralnetworks::V1_2::implementation { - -using V1_0::ErrorStatus; - -constexpr Timing kNoTiming = {.timeOnDevice = std::numeric_limits::max(), - .timeInDriver = std::numeric_limits::max()}; - -// PreparedModelCallback methods begin here - -Return PreparedModelCallback::notify(ErrorStatus errorStatus, - const sp& preparedModel) { - { - std::lock_guard hold(mMutex); - - // quick-return if object has already been notified - if (mNotified) { - return Void(); - } - - // store results and mark as notified - mErrorStatus = errorStatus; - mPreparedModel = preparedModel; - mNotified = true; - } - - mCondition.notify_all(); - return Void(); -} - -Return PreparedModelCallback::notify_1_2(ErrorStatus errorStatus, - const sp& preparedModel) { - return notify(errorStatus, preparedModel); -} - -void PreparedModelCallback::wait() const { - std::unique_lock lock(mMutex); - mCondition.wait(lock, [this] { return mNotified; }); -} - -ErrorStatus PreparedModelCallback::getStatus() const { - wait(); - return mErrorStatus; -} - -sp PreparedModelCallback::getPreparedModel() const { - wait(); - return mPreparedModel; -} - -// ExecutionCallback methods begin here - -Return ExecutionCallback::notify(ErrorStatus errorStatus) { - notifyInternal(errorStatus, {}, kNoTiming); - return Void(); -} - -Return ExecutionCallback::notify_1_2(ErrorStatus errorStatus, - const hidl_vec& outputShapes, - const Timing& timing) { - if (errorStatus == ErrorStatus::OUTPUT_INSUFFICIENT_SIZE) { - // outputShapes must not be empty if OUTPUT_INSUFFICIENT_SIZE. - if (outputShapes.size() == 0) { - LOG(ERROR) << "Notified with empty output shape vector when OUTPUT_INSUFFICIENT_SIZE"; - notifyInternal(ErrorStatus::GENERAL_FAILURE, {}, kNoTiming); - return Void(); - } - } else if (errorStatus != ErrorStatus::NONE) { - // outputShapes must be empty if errorStatus is neither NONE nor OUTPUT_INSUFFICIENT_SIZE. - if (outputShapes.size() != 0) { - LOG(ERROR) << "Notified with non-empty output shape vector when error status is " - "neither NONE nor OUTPUT_INSUFFICIENT_SIZE"; - notifyInternal(ErrorStatus::GENERAL_FAILURE, {}, kNoTiming); - return Void(); - } - } - notifyInternal(errorStatus, outputShapes, timing); - return Void(); -} - -void ExecutionCallback::wait() const { - std::unique_lock lock(mMutex); - mCondition.wait(lock, [this] { return mNotified; }); -} - -ErrorStatus ExecutionCallback::getStatus() const { - wait(); - return mErrorStatus; -} - -const std::vector& ExecutionCallback::getOutputShapes() const { - wait(); - return mOutputShapes; -} - -Timing ExecutionCallback::getTiming() const { - wait(); - return mTiming; -} - -void ExecutionCallback::notifyInternal(ErrorStatus errorStatus, - const hidl_vec& outputShapes, - const Timing& timing) { - { - std::lock_guard hold(mMutex); - - // quick-return if object has already been notified - if (mNotified) { - return; - } - - mErrorStatus = errorStatus; - mOutputShapes = outputShapes; - mTiming = timing; - mNotified = true; - } - mCondition.notify_all(); -} - -} // namespace android::hardware::neuralnetworks::V1_2::implementation diff --git a/neuralnetworks/1.3/vts/functional/CompilationCachingTests.cpp b/neuralnetworks/1.3/vts/functional/CompilationCachingTests.cpp deleted file mode 100644 index 2130a76b75..0000000000 --- a/neuralnetworks/1.3/vts/functional/CompilationCachingTests.cpp +++ /dev/null @@ -1,1374 +0,0 @@ -/* - * Copyright (C) 2019 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#define LOG_TAG "neuralnetworks_hidl_hal_test" - -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include - -#include "1.2/Callbacks.h" -#include "GeneratedTestHarness.h" -#include "MemoryUtils.h" -#include "TestHarness.h" -#include "Utils.h" -#include "VtsHalNeuralnetworks.h" - -// Forward declaration of the mobilenet generated test models in -// frameworks/ml/nn/runtime/test/generated/. -namespace generated_tests::mobilenet_224_gender_basic_fixed { -const test_helper::TestModel& get_test_model(); -} // namespace generated_tests::mobilenet_224_gender_basic_fixed - -namespace generated_tests::mobilenet_quantized { -const test_helper::TestModel& get_test_model(); -} // namespace generated_tests::mobilenet_quantized - -namespace android::hardware::neuralnetworks::V1_2::vts::functional { - -using namespace test_helper; -using implementation::PreparedModelCallback; -using V1_0::ErrorStatus; -using V1_1::ExecutionPreference; - -namespace float32_model { - -constexpr auto get_test_model = generated_tests::mobilenet_224_gender_basic_fixed::get_test_model; - -} // namespace float32_model - -namespace quant8_model { - -constexpr auto get_test_model = generated_tests::mobilenet_quantized::get_test_model; - -} // namespace quant8_model - -namespace { - -enum class AccessMode { READ_WRITE, READ_ONLY, WRITE_ONLY }; - -// Creates cache handles based on provided file groups. -// The outer vector corresponds to handles and the inner vector is for fds held by each handle. -void createCacheHandles(const std::vector>& fileGroups, - const std::vector& mode, hidl_vec* handles) { - handles->resize(fileGroups.size()); - for (uint32_t i = 0; i < fileGroups.size(); i++) { - std::vector fds; - for (const auto& file : fileGroups[i]) { - int fd; - if (mode[i] == AccessMode::READ_ONLY) { - fd = open(file.c_str(), O_RDONLY); - } else if (mode[i] == AccessMode::WRITE_ONLY) { - fd = open(file.c_str(), O_WRONLY | O_CREAT, S_IRUSR | S_IWUSR); - } else if (mode[i] == AccessMode::READ_WRITE) { - fd = open(file.c_str(), O_RDWR | O_CREAT, S_IRUSR | S_IWUSR); - } else { - FAIL(); - } - ASSERT_GE(fd, 0); - fds.push_back(fd); - } - native_handle_t* cacheNativeHandle = native_handle_create(fds.size(), 0); - ASSERT_NE(cacheNativeHandle, nullptr); - std::copy(fds.begin(), fds.end(), &cacheNativeHandle->data[0]); - (*handles)[i].setTo(cacheNativeHandle, /*shouldOwn=*/true); - } -} - -void createCacheHandles(const std::vector>& fileGroups, AccessMode mode, - hidl_vec* handles) { - createCacheHandles(fileGroups, std::vector(fileGroups.size(), mode), handles); -} - -// Create a chain of broadcast operations. The second operand is always constant tensor [1]. -// For simplicity, activation scalar is shared. The second operand is not shared -// in the model to let driver maintain a non-trivial size of constant data and the corresponding -// data locations in cache. -// -// --------- activation -------- -// ↓ ↓ ↓ ↓ -// E.g. input -> ADD -> ADD -> ADD -> ... -> ADD -> output -// ↑ ↑ ↑ ↑ -// [1] [1] [1] [1] -// -// This function assumes the operation is either ADD or MUL. -template -TestModel createLargeTestModelImpl(TestOperationType op, uint32_t len) { - EXPECT_TRUE(op == TestOperationType::ADD || op == TestOperationType::MUL); - - // Model operations and operands. - std::vector operations(len); - std::vector operands(len * 2 + 2); - - // The activation scalar, value = 0. - operands[0] = { - .type = TestOperandType::INT32, - .dimensions = {}, - .numberOfConsumers = len, - .scale = 0.0f, - .zeroPoint = 0, - .lifetime = TestOperandLifeTime::CONSTANT_COPY, - .data = TestBuffer::createFromVector({0}), - }; - - // The buffer value of the constant second operand. The logical value is always 1.0f. - CppType bufferValue; - // The scale of the first and second operand. - float scale1, scale2; - if (operandType == TestOperandType::TENSOR_FLOAT32) { - bufferValue = 1.0f; - scale1 = 0.0f; - scale2 = 0.0f; - } else if (op == TestOperationType::ADD) { - bufferValue = 1; - scale1 = 1.0f; - scale2 = 1.0f; - } else { - // To satisfy the constraint on quant8 MUL: input0.scale * input1.scale < output.scale, - // set input1 to have scale = 0.5f and bufferValue = 2, i.e. 1.0f in floating point. - bufferValue = 2; - scale1 = 1.0f; - scale2 = 0.5f; - } - - for (uint32_t i = 0; i < len; i++) { - const uint32_t firstInputIndex = i * 2 + 1; - const uint32_t secondInputIndex = firstInputIndex + 1; - const uint32_t outputIndex = secondInputIndex + 1; - - // The first operation input. - operands[firstInputIndex] = { - .type = operandType, - .dimensions = {1}, - .numberOfConsumers = 1, - .scale = scale1, - .zeroPoint = 0, - .lifetime = (i == 0 ? TestOperandLifeTime::MODEL_INPUT - : TestOperandLifeTime::TEMPORARY_VARIABLE), - .data = (i == 0 ? TestBuffer::createFromVector({1}) : TestBuffer()), - }; - - // The second operation input, value = 1. - operands[secondInputIndex] = { - .type = operandType, - .dimensions = {1}, - .numberOfConsumers = 1, - .scale = scale2, - .zeroPoint = 0, - .lifetime = TestOperandLifeTime::CONSTANT_COPY, - .data = TestBuffer::createFromVector({bufferValue}), - }; - - // The operation. All operations share the same activation scalar. - // The output operand is created as an input in the next iteration of the loop, in the case - // of all but the last member of the chain; and after the loop as a model output, in the - // case of the last member of the chain. - operations[i] = { - .type = op, - .inputs = {firstInputIndex, secondInputIndex, /*activation scalar*/ 0}, - .outputs = {outputIndex}, - }; - } - - // For TestOperationType::ADD, output = 1 + 1 * len = len + 1 - // For TestOperationType::MUL, output = 1 * 1 ^ len = 1 - CppType outputResult = static_cast(op == TestOperationType::ADD ? len + 1u : 1u); - - // The model output. - operands.back() = { - .type = operandType, - .dimensions = {1}, - .numberOfConsumers = 0, - .scale = scale1, - .zeroPoint = 0, - .lifetime = TestOperandLifeTime::MODEL_OUTPUT, - .data = TestBuffer::createFromVector({outputResult}), - }; - - return { - .operands = std::move(operands), - .operations = std::move(operations), - .inputIndexes = {1}, - .outputIndexes = {len * 2 + 1}, - .isRelaxed = false, - }; -} - -} // namespace - -// Tag for the compilation caching tests. -class CompilationCachingTestBase : public testing::Test { - protected: - CompilationCachingTestBase(sp device, OperandType type) - : kDevice(std::move(device)), kOperandType(type) {} - - void SetUp() override { - testing::Test::SetUp(); - ASSERT_NE(kDevice.get(), nullptr); - - // Create cache directory. The cache directory and a temporary cache file is always created - // to test the behavior of prepareModelFromCache, even when caching is not supported. - char cacheDirTemp[] = "/data/local/tmp/TestCompilationCachingXXXXXX"; - char* cacheDir = mkdtemp(cacheDirTemp); - ASSERT_NE(cacheDir, nullptr); - mCacheDir = cacheDir; - mCacheDir.push_back('/'); - - Return ret = kDevice->getNumberOfCacheFilesNeeded( - [this](ErrorStatus status, uint32_t numModelCache, uint32_t numDataCache) { - EXPECT_EQ(ErrorStatus::NONE, status); - mNumModelCache = numModelCache; - mNumDataCache = numDataCache; - }); - EXPECT_TRUE(ret.isOk()); - mIsCachingSupported = mNumModelCache > 0 || mNumDataCache > 0; - - // Create empty cache files. - mTmpCache = mCacheDir + "tmp"; - for (uint32_t i = 0; i < mNumModelCache; i++) { - mModelCache.push_back({mCacheDir + "model" + std::to_string(i)}); - } - for (uint32_t i = 0; i < mNumDataCache; i++) { - mDataCache.push_back({mCacheDir + "data" + std::to_string(i)}); - } - // Dummy handles, use AccessMode::WRITE_ONLY for createCacheHandles to create files. - hidl_vec modelHandle, dataHandle, tmpHandle; - createCacheHandles(mModelCache, AccessMode::WRITE_ONLY, &modelHandle); - createCacheHandles(mDataCache, AccessMode::WRITE_ONLY, &dataHandle); - createCacheHandles({{mTmpCache}}, AccessMode::WRITE_ONLY, &tmpHandle); - - if (!mIsCachingSupported) { - LOG(INFO) << "NN VTS: Early termination of test because vendor service does not " - "support compilation caching."; - std::cout << "[ ] Early termination of test because vendor service does not " - "support compilation caching." - << std::endl; - } - } - - void TearDown() override { - // If the test passes, remove the tmp directory. Otherwise, keep it for debugging purposes. - if (!testing::Test::HasFailure()) { - // Recursively remove the cache directory specified by mCacheDir. - auto callback = [](const char* entry, const struct stat*, int, struct FTW*) { - return remove(entry); - }; - nftw(mCacheDir.c_str(), callback, 128, FTW_DEPTH | FTW_MOUNT | FTW_PHYS); - } - testing::Test::TearDown(); - } - - // Model and examples creators. According to kOperandType, the following methods will return - // either float32 model/examples or the quant8 variant. - TestModel createTestModel() { - if (kOperandType == OperandType::TENSOR_FLOAT32) { - return float32_model::get_test_model(); - } else { - return quant8_model::get_test_model(); - } - } - - TestModel createLargeTestModel(OperationType op, uint32_t len) { - if (kOperandType == OperandType::TENSOR_FLOAT32) { - return createLargeTestModelImpl( - static_cast(op), len); - } else { - return createLargeTestModelImpl( - static_cast(op), len); - } - } - - // See if the service can handle the model. - bool isModelFullySupported(const Model& model) { - bool fullySupportsModel = false; - Return supportedCall = kDevice->getSupportedOperations_1_2( - model, - [&fullySupportsModel, &model](ErrorStatus status, const hidl_vec& supported) { - ASSERT_EQ(ErrorStatus::NONE, status); - ASSERT_EQ(supported.size(), model.operations.size()); - fullySupportsModel = std::all_of(supported.begin(), supported.end(), - [](bool valid) { return valid; }); - }); - EXPECT_TRUE(supportedCall.isOk()); - return fullySupportsModel; - } - - void saveModelToCache(const Model& model, const hidl_vec& modelCache, - const hidl_vec& dataCache, - sp* preparedModel = nullptr) { - if (preparedModel != nullptr) *preparedModel = nullptr; - - // Launch prepare model. - sp preparedModelCallback = new PreparedModelCallback(); - hidl_array cacheToken(mToken); - Return prepareLaunchStatus = - kDevice->prepareModel_1_2(model, ExecutionPreference::FAST_SINGLE_ANSWER, - modelCache, dataCache, cacheToken, preparedModelCallback); - ASSERT_TRUE(prepareLaunchStatus.isOk()); - ASSERT_EQ(static_cast(prepareLaunchStatus), ErrorStatus::NONE); - - // Retrieve prepared model. - preparedModelCallback->wait(); - ASSERT_EQ(preparedModelCallback->getStatus(), ErrorStatus::NONE); - if (preparedModel != nullptr) { - *preparedModel = IPreparedModel::castFrom(preparedModelCallback->getPreparedModel()) - .withDefault(nullptr); - } - } - - bool checkEarlyTermination(ErrorStatus status) { - if (status == ErrorStatus::GENERAL_FAILURE) { - LOG(INFO) << "NN VTS: Early termination of test because vendor service cannot " - "save the prepared model that it does not support."; - std::cout << "[ ] Early termination of test because vendor service cannot " - "save the prepared model that it does not support." - << std::endl; - return true; - } - return false; - } - - bool checkEarlyTermination(const Model& model) { - if (!isModelFullySupported(model)) { - LOG(INFO) << "NN VTS: Early termination of test because vendor service cannot " - "prepare model that it does not support."; - std::cout << "[ ] Early termination of test because vendor service cannot " - "prepare model that it does not support." - << std::endl; - return true; - } - return false; - } - - void prepareModelFromCache(const hidl_vec& modelCache, - const hidl_vec& dataCache, - sp* preparedModel, ErrorStatus* status) { - // Launch prepare model from cache. - sp preparedModelCallback = new PreparedModelCallback(); - hidl_array cacheToken(mToken); - Return prepareLaunchStatus = kDevice->prepareModelFromCache( - modelCache, dataCache, cacheToken, preparedModelCallback); - ASSERT_TRUE(prepareLaunchStatus.isOk()); - if (static_cast(prepareLaunchStatus) != ErrorStatus::NONE) { - *preparedModel = nullptr; - *status = static_cast(prepareLaunchStatus); - return; - } - - // Retrieve prepared model. - preparedModelCallback->wait(); - *status = preparedModelCallback->getStatus(); - *preparedModel = IPreparedModel::castFrom(preparedModelCallback->getPreparedModel()) - .withDefault(nullptr); - } - - // Absolute path to the temporary cache directory. - std::string mCacheDir; - - // Groups of file paths for model and data cache in the tmp cache directory, initialized with - // outer_size = mNum{Model|Data}Cache, inner_size = 1. The outer vector corresponds to handles - // and the inner vector is for fds held by each handle. - std::vector> mModelCache; - std::vector> mDataCache; - - // A separate temporary file path in the tmp cache directory. - std::string mTmpCache; - - uint8_t mToken[static_cast(Constant::BYTE_SIZE_OF_CACHE_TOKEN)] = {}; - uint32_t mNumModelCache; - uint32_t mNumDataCache; - uint32_t mIsCachingSupported; - - const sp kDevice; - // The primary data type of the testModel. - const OperandType kOperandType; -}; - -using CompilationCachingTestParam = std::tuple; - -// A parameterized fixture of CompilationCachingTestBase. Every test will run twice, with the first -// pass running with float32 models and the second pass running with quant8 models. -class CompilationCachingTest : public CompilationCachingTestBase, - public testing::WithParamInterface { - protected: - CompilationCachingTest() - : CompilationCachingTestBase(getData(std::get(GetParam())), - std::get(GetParam())) {} -}; - -TEST_P(CompilationCachingTest, CacheSavingAndRetrieval) { - // Create test HIDL model and compile. - const TestModel& testModel = createTestModel(); - const Model model = createModel(testModel); - if (checkEarlyTermination(model)) return; - sp preparedModel = nullptr; - - // Save the compilation to cache. - { - hidl_vec modelCache, dataCache; - createCacheHandles(mModelCache, AccessMode::READ_WRITE, &modelCache); - createCacheHandles(mDataCache, AccessMode::READ_WRITE, &dataCache); - saveModelToCache(model, modelCache, dataCache); - } - - // Retrieve preparedModel from cache. - { - preparedModel = nullptr; - ErrorStatus status; - hidl_vec modelCache, dataCache; - createCacheHandles(mModelCache, AccessMode::READ_WRITE, &modelCache); - createCacheHandles(mDataCache, AccessMode::READ_WRITE, &dataCache); - prepareModelFromCache(modelCache, dataCache, &preparedModel, &status); - if (!mIsCachingSupported) { - ASSERT_EQ(status, ErrorStatus::GENERAL_FAILURE); - ASSERT_EQ(preparedModel, nullptr); - return; - } else if (checkEarlyTermination(status)) { - ASSERT_EQ(preparedModel, nullptr); - return; - } else { - ASSERT_EQ(status, ErrorStatus::NONE); - ASSERT_NE(preparedModel, nullptr); - } - } - - // Execute and verify results. - EvaluatePreparedModel(preparedModel, testModel, - /*testDynamicOutputShape=*/false); -} - -TEST_P(CompilationCachingTest, CacheSavingAndRetrievalNonZeroOffset) { - // Create test HIDL model and compile. - const TestModel& testModel = createTestModel(); - const Model model = createModel(testModel); - if (checkEarlyTermination(model)) return; - sp preparedModel = nullptr; - - // Save the compilation to cache. - { - hidl_vec modelCache, dataCache; - createCacheHandles(mModelCache, AccessMode::READ_WRITE, &modelCache); - createCacheHandles(mDataCache, AccessMode::READ_WRITE, &dataCache); - uint8_t dummyBytes[] = {0, 0}; - // Write a dummy integer to the cache. - // The driver should be able to handle non-empty cache and non-zero fd offset. - for (uint32_t i = 0; i < modelCache.size(); i++) { - ASSERT_EQ(write(modelCache[i].getNativeHandle()->data[0], &dummyBytes, - sizeof(dummyBytes)), - sizeof(dummyBytes)); - } - for (uint32_t i = 0; i < dataCache.size(); i++) { - ASSERT_EQ( - write(dataCache[i].getNativeHandle()->data[0], &dummyBytes, sizeof(dummyBytes)), - sizeof(dummyBytes)); - } - saveModelToCache(model, modelCache, dataCache); - } - - // Retrieve preparedModel from cache. - { - preparedModel = nullptr; - ErrorStatus status; - hidl_vec modelCache, dataCache; - createCacheHandles(mModelCache, AccessMode::READ_WRITE, &modelCache); - createCacheHandles(mDataCache, AccessMode::READ_WRITE, &dataCache); - uint8_t dummyByte = 0; - // Advance the offset of each handle by one byte. - // The driver should be able to handle non-zero fd offset. - for (uint32_t i = 0; i < modelCache.size(); i++) { - ASSERT_GE(read(modelCache[i].getNativeHandle()->data[0], &dummyByte, 1), 0); - } - for (uint32_t i = 0; i < dataCache.size(); i++) { - ASSERT_GE(read(dataCache[i].getNativeHandle()->data[0], &dummyByte, 1), 0); - } - prepareModelFromCache(modelCache, dataCache, &preparedModel, &status); - if (!mIsCachingSupported) { - ASSERT_EQ(status, ErrorStatus::GENERAL_FAILURE); - ASSERT_EQ(preparedModel, nullptr); - return; - } else if (checkEarlyTermination(status)) { - ASSERT_EQ(preparedModel, nullptr); - return; - } else { - ASSERT_EQ(status, ErrorStatus::NONE); - ASSERT_NE(preparedModel, nullptr); - } - } - - // Execute and verify results. - EvaluatePreparedModel(preparedModel, testModel, - /*testDynamicOutputShape=*/false); -} - -TEST_P(CompilationCachingTest, SaveToCacheInvalidNumCache) { - // Create test HIDL model and compile. - const TestModel& testModel = createTestModel(); - const Model model = createModel(testModel); - if (checkEarlyTermination(model)) return; - - // Test with number of model cache files greater than mNumModelCache. - { - hidl_vec modelCache, dataCache; - // Pass an additional cache file for model cache. - mModelCache.push_back({mTmpCache}); - createCacheHandles(mModelCache, AccessMode::READ_WRITE, &modelCache); - createCacheHandles(mDataCache, AccessMode::READ_WRITE, &dataCache); - mModelCache.pop_back(); - sp preparedModel = nullptr; - saveModelToCache(model, modelCache, dataCache, &preparedModel); - ASSERT_NE(preparedModel, nullptr); - // Execute and verify results. - EvaluatePreparedModel(preparedModel, testModel, - /*testDynamicOutputShape=*/false); - // Check if prepareModelFromCache fails. - preparedModel = nullptr; - ErrorStatus status; - prepareModelFromCache(modelCache, dataCache, &preparedModel, &status); - if (status != ErrorStatus::INVALID_ARGUMENT) { - ASSERT_EQ(status, ErrorStatus::GENERAL_FAILURE); - } - ASSERT_EQ(preparedModel, nullptr); - } - - // Test with number of model cache files smaller than mNumModelCache. - if (mModelCache.size() > 0) { - hidl_vec modelCache, dataCache; - // Pop out the last cache file. - auto tmp = mModelCache.back(); - mModelCache.pop_back(); - createCacheHandles(mModelCache, AccessMode::READ_WRITE, &modelCache); - createCacheHandles(mDataCache, AccessMode::READ_WRITE, &dataCache); - mModelCache.push_back(tmp); - sp preparedModel = nullptr; - saveModelToCache(model, modelCache, dataCache, &preparedModel); - ASSERT_NE(preparedModel, nullptr); - // Execute and verify results. - EvaluatePreparedModel(preparedModel, testModel, - /*testDynamicOutputShape=*/false); - // Check if prepareModelFromCache fails. - preparedModel = nullptr; - ErrorStatus status; - prepareModelFromCache(modelCache, dataCache, &preparedModel, &status); - if (status != ErrorStatus::INVALID_ARGUMENT) { - ASSERT_EQ(status, ErrorStatus::GENERAL_FAILURE); - } - ASSERT_EQ(preparedModel, nullptr); - } - - // Test with number of data cache files greater than mNumDataCache. - { - hidl_vec modelCache, dataCache; - // Pass an additional cache file for data cache. - mDataCache.push_back({mTmpCache}); - createCacheHandles(mModelCache, AccessMode::READ_WRITE, &modelCache); - createCacheHandles(mDataCache, AccessMode::READ_WRITE, &dataCache); - mDataCache.pop_back(); - sp preparedModel = nullptr; - saveModelToCache(model, modelCache, dataCache, &preparedModel); - ASSERT_NE(preparedModel, nullptr); - // Execute and verify results. - EvaluatePreparedModel(preparedModel, testModel, - /*testDynamicOutputShape=*/false); - // Check if prepareModelFromCache fails. - preparedModel = nullptr; - ErrorStatus status; - prepareModelFromCache(modelCache, dataCache, &preparedModel, &status); - if (status != ErrorStatus::INVALID_ARGUMENT) { - ASSERT_EQ(status, ErrorStatus::GENERAL_FAILURE); - } - ASSERT_EQ(preparedModel, nullptr); - } - - // Test with number of data cache files smaller than mNumDataCache. - if (mDataCache.size() > 0) { - hidl_vec modelCache, dataCache; - // Pop out the last cache file. - auto tmp = mDataCache.back(); - mDataCache.pop_back(); - createCacheHandles(mModelCache, AccessMode::READ_WRITE, &modelCache); - createCacheHandles(mDataCache, AccessMode::READ_WRITE, &dataCache); - mDataCache.push_back(tmp); - sp preparedModel = nullptr; - saveModelToCache(model, modelCache, dataCache, &preparedModel); - ASSERT_NE(preparedModel, nullptr); - // Execute and verify results. - EvaluatePreparedModel(preparedModel, testModel, - /*testDynamicOutputShape=*/false); - // Check if prepareModelFromCache fails. - preparedModel = nullptr; - ErrorStatus status; - prepareModelFromCache(modelCache, dataCache, &preparedModel, &status); - if (status != ErrorStatus::INVALID_ARGUMENT) { - ASSERT_EQ(status, ErrorStatus::GENERAL_FAILURE); - } - ASSERT_EQ(preparedModel, nullptr); - } -} - -TEST_P(CompilationCachingTest, PrepareModelFromCacheInvalidNumCache) { - // Create test HIDL model and compile. - const TestModel& testModel = createTestModel(); - const Model model = createModel(testModel); - if (checkEarlyTermination(model)) return; - - // Save the compilation to cache. - { - hidl_vec modelCache, dataCache; - createCacheHandles(mModelCache, AccessMode::READ_WRITE, &modelCache); - createCacheHandles(mDataCache, AccessMode::READ_WRITE, &dataCache); - saveModelToCache(model, modelCache, dataCache); - } - - // Test with number of model cache files greater than mNumModelCache. - { - sp preparedModel = nullptr; - ErrorStatus status; - hidl_vec modelCache, dataCache; - mModelCache.push_back({mTmpCache}); - createCacheHandles(mModelCache, AccessMode::READ_WRITE, &modelCache); - createCacheHandles(mDataCache, AccessMode::READ_WRITE, &dataCache); - mModelCache.pop_back(); - prepareModelFromCache(modelCache, dataCache, &preparedModel, &status); - if (status != ErrorStatus::GENERAL_FAILURE) { - ASSERT_EQ(status, ErrorStatus::INVALID_ARGUMENT); - } - ASSERT_EQ(preparedModel, nullptr); - } - - // Test with number of model cache files smaller than mNumModelCache. - if (mModelCache.size() > 0) { - sp preparedModel = nullptr; - ErrorStatus status; - hidl_vec modelCache, dataCache; - auto tmp = mModelCache.back(); - mModelCache.pop_back(); - createCacheHandles(mModelCache, AccessMode::READ_WRITE, &modelCache); - createCacheHandles(mDataCache, AccessMode::READ_WRITE, &dataCache); - mModelCache.push_back(tmp); - prepareModelFromCache(modelCache, dataCache, &preparedModel, &status); - if (status != ErrorStatus::GENERAL_FAILURE) { - ASSERT_EQ(status, ErrorStatus::INVALID_ARGUMENT); - } - ASSERT_EQ(preparedModel, nullptr); - } - - // Test with number of data cache files greater than mNumDataCache. - { - sp preparedModel = nullptr; - ErrorStatus status; - hidl_vec modelCache, dataCache; - mDataCache.push_back({mTmpCache}); - createCacheHandles(mModelCache, AccessMode::READ_WRITE, &modelCache); - createCacheHandles(mDataCache, AccessMode::READ_WRITE, &dataCache); - mDataCache.pop_back(); - prepareModelFromCache(modelCache, dataCache, &preparedModel, &status); - if (status != ErrorStatus::GENERAL_FAILURE) { - ASSERT_EQ(status, ErrorStatus::INVALID_ARGUMENT); - } - ASSERT_EQ(preparedModel, nullptr); - } - - // Test with number of data cache files smaller than mNumDataCache. - if (mDataCache.size() > 0) { - sp preparedModel = nullptr; - ErrorStatus status; - hidl_vec modelCache, dataCache; - auto tmp = mDataCache.back(); - mDataCache.pop_back(); - createCacheHandles(mModelCache, AccessMode::READ_WRITE, &modelCache); - createCacheHandles(mDataCache, AccessMode::READ_WRITE, &dataCache); - mDataCache.push_back(tmp); - prepareModelFromCache(modelCache, dataCache, &preparedModel, &status); - if (status != ErrorStatus::GENERAL_FAILURE) { - ASSERT_EQ(status, ErrorStatus::INVALID_ARGUMENT); - } - ASSERT_EQ(preparedModel, nullptr); - } -} - -TEST_P(CompilationCachingTest, SaveToCacheInvalidNumFd) { - // Create test HIDL model and compile. - const TestModel& testModel = createTestModel(); - const Model model = createModel(testModel); - if (checkEarlyTermination(model)) return; - - // Go through each handle in model cache, test with NumFd greater than 1. - for (uint32_t i = 0; i < mNumModelCache; i++) { - hidl_vec modelCache, dataCache; - // Pass an invalid number of fds for handle i. - mModelCache[i].push_back(mTmpCache); - createCacheHandles(mModelCache, AccessMode::READ_WRITE, &modelCache); - createCacheHandles(mDataCache, AccessMode::READ_WRITE, &dataCache); - mModelCache[i].pop_back(); - sp preparedModel = nullptr; - saveModelToCache(model, modelCache, dataCache, &preparedModel); - ASSERT_NE(preparedModel, nullptr); - // Execute and verify results. - EvaluatePreparedModel(preparedModel, testModel, - /*testDynamicOutputShape=*/false); - // Check if prepareModelFromCache fails. - preparedModel = nullptr; - ErrorStatus status; - prepareModelFromCache(modelCache, dataCache, &preparedModel, &status); - if (status != ErrorStatus::INVALID_ARGUMENT) { - ASSERT_EQ(status, ErrorStatus::GENERAL_FAILURE); - } - ASSERT_EQ(preparedModel, nullptr); - } - - // Go through each handle in model cache, test with NumFd equal to 0. - for (uint32_t i = 0; i < mNumModelCache; i++) { - hidl_vec modelCache, dataCache; - // Pass an invalid number of fds for handle i. - auto tmp = mModelCache[i].back(); - mModelCache[i].pop_back(); - createCacheHandles(mModelCache, AccessMode::READ_WRITE, &modelCache); - createCacheHandles(mDataCache, AccessMode::READ_WRITE, &dataCache); - mModelCache[i].push_back(tmp); - sp preparedModel = nullptr; - saveModelToCache(model, modelCache, dataCache, &preparedModel); - ASSERT_NE(preparedModel, nullptr); - // Execute and verify results. - EvaluatePreparedModel(preparedModel, testModel, - /*testDynamicOutputShape=*/false); - // Check if prepareModelFromCache fails. - preparedModel = nullptr; - ErrorStatus status; - prepareModelFromCache(modelCache, dataCache, &preparedModel, &status); - if (status != ErrorStatus::INVALID_ARGUMENT) { - ASSERT_EQ(status, ErrorStatus::GENERAL_FAILURE); - } - ASSERT_EQ(preparedModel, nullptr); - } - - // Go through each handle in data cache, test with NumFd greater than 1. - for (uint32_t i = 0; i < mNumDataCache; i++) { - hidl_vec modelCache, dataCache; - // Pass an invalid number of fds for handle i. - mDataCache[i].push_back(mTmpCache); - createCacheHandles(mModelCache, AccessMode::READ_WRITE, &modelCache); - createCacheHandles(mDataCache, AccessMode::READ_WRITE, &dataCache); - mDataCache[i].pop_back(); - sp preparedModel = nullptr; - saveModelToCache(model, modelCache, dataCache, &preparedModel); - ASSERT_NE(preparedModel, nullptr); - // Execute and verify results. - EvaluatePreparedModel(preparedModel, testModel, - /*testDynamicOutputShape=*/false); - // Check if prepareModelFromCache fails. - preparedModel = nullptr; - ErrorStatus status; - prepareModelFromCache(modelCache, dataCache, &preparedModel, &status); - if (status != ErrorStatus::INVALID_ARGUMENT) { - ASSERT_EQ(status, ErrorStatus::GENERAL_FAILURE); - } - ASSERT_EQ(preparedModel, nullptr); - } - - // Go through each handle in data cache, test with NumFd equal to 0. - for (uint32_t i = 0; i < mNumDataCache; i++) { - hidl_vec modelCache, dataCache; - // Pass an invalid number of fds for handle i. - auto tmp = mDataCache[i].back(); - mDataCache[i].pop_back(); - createCacheHandles(mModelCache, AccessMode::READ_WRITE, &modelCache); - createCacheHandles(mDataCache, AccessMode::READ_WRITE, &dataCache); - mDataCache[i].push_back(tmp); - sp preparedModel = nullptr; - saveModelToCache(model, modelCache, dataCache, &preparedModel); - ASSERT_NE(preparedModel, nullptr); - // Execute and verify results. - EvaluatePreparedModel(preparedModel, testModel, - /*testDynamicOutputShape=*/false); - // Check if prepareModelFromCache fails. - preparedModel = nullptr; - ErrorStatus status; - prepareModelFromCache(modelCache, dataCache, &preparedModel, &status); - if (status != ErrorStatus::INVALID_ARGUMENT) { - ASSERT_EQ(status, ErrorStatus::GENERAL_FAILURE); - } - ASSERT_EQ(preparedModel, nullptr); - } -} - -TEST_P(CompilationCachingTest, PrepareModelFromCacheInvalidNumFd) { - // Create test HIDL model and compile. - const TestModel& testModel = createTestModel(); - const Model model = createModel(testModel); - if (checkEarlyTermination(model)) return; - - // Save the compilation to cache. - { - hidl_vec modelCache, dataCache; - createCacheHandles(mModelCache, AccessMode::READ_WRITE, &modelCache); - createCacheHandles(mDataCache, AccessMode::READ_WRITE, &dataCache); - saveModelToCache(model, modelCache, dataCache); - } - - // Go through each handle in model cache, test with NumFd greater than 1. - for (uint32_t i = 0; i < mNumModelCache; i++) { - sp preparedModel = nullptr; - ErrorStatus status; - hidl_vec modelCache, dataCache; - mModelCache[i].push_back(mTmpCache); - createCacheHandles(mModelCache, AccessMode::READ_WRITE, &modelCache); - createCacheHandles(mDataCache, AccessMode::READ_WRITE, &dataCache); - mModelCache[i].pop_back(); - prepareModelFromCache(modelCache, dataCache, &preparedModel, &status); - if (status != ErrorStatus::GENERAL_FAILURE) { - ASSERT_EQ(status, ErrorStatus::INVALID_ARGUMENT); - } - ASSERT_EQ(preparedModel, nullptr); - } - - // Go through each handle in model cache, test with NumFd equal to 0. - for (uint32_t i = 0; i < mNumModelCache; i++) { - sp preparedModel = nullptr; - ErrorStatus status; - hidl_vec modelCache, dataCache; - auto tmp = mModelCache[i].back(); - mModelCache[i].pop_back(); - createCacheHandles(mModelCache, AccessMode::READ_WRITE, &modelCache); - createCacheHandles(mDataCache, AccessMode::READ_WRITE, &dataCache); - mModelCache[i].push_back(tmp); - prepareModelFromCache(modelCache, dataCache, &preparedModel, &status); - if (status != ErrorStatus::GENERAL_FAILURE) { - ASSERT_EQ(status, ErrorStatus::INVALID_ARGUMENT); - } - ASSERT_EQ(preparedModel, nullptr); - } - - // Go through each handle in data cache, test with NumFd greater than 1. - for (uint32_t i = 0; i < mNumDataCache; i++) { - sp preparedModel = nullptr; - ErrorStatus status; - hidl_vec modelCache, dataCache; - mDataCache[i].push_back(mTmpCache); - createCacheHandles(mModelCache, AccessMode::READ_WRITE, &modelCache); - createCacheHandles(mDataCache, AccessMode::READ_WRITE, &dataCache); - mDataCache[i].pop_back(); - prepareModelFromCache(modelCache, dataCache, &preparedModel, &status); - if (status != ErrorStatus::GENERAL_FAILURE) { - ASSERT_EQ(status, ErrorStatus::INVALID_ARGUMENT); - } - ASSERT_EQ(preparedModel, nullptr); - } - - // Go through each handle in data cache, test with NumFd equal to 0. - for (uint32_t i = 0; i < mNumDataCache; i++) { - sp preparedModel = nullptr; - ErrorStatus status; - hidl_vec modelCache, dataCache; - auto tmp = mDataCache[i].back(); - mDataCache[i].pop_back(); - createCacheHandles(mModelCache, AccessMode::READ_WRITE, &modelCache); - createCacheHandles(mDataCache, AccessMode::READ_WRITE, &dataCache); - mDataCache[i].push_back(tmp); - prepareModelFromCache(modelCache, dataCache, &preparedModel, &status); - if (status != ErrorStatus::GENERAL_FAILURE) { - ASSERT_EQ(status, ErrorStatus::INVALID_ARGUMENT); - } - ASSERT_EQ(preparedModel, nullptr); - } -} - -TEST_P(CompilationCachingTest, SaveToCacheInvalidAccessMode) { - // Create test HIDL model and compile. - const TestModel& testModel = createTestModel(); - const Model model = createModel(testModel); - if (checkEarlyTermination(model)) return; - std::vector modelCacheMode(mNumModelCache, AccessMode::READ_WRITE); - std::vector dataCacheMode(mNumDataCache, AccessMode::READ_WRITE); - - // Go through each handle in model cache, test with invalid access mode. - for (uint32_t i = 0; i < mNumModelCache; i++) { - hidl_vec modelCache, dataCache; - modelCacheMode[i] = AccessMode::READ_ONLY; - createCacheHandles(mModelCache, modelCacheMode, &modelCache); - createCacheHandles(mDataCache, dataCacheMode, &dataCache); - modelCacheMode[i] = AccessMode::READ_WRITE; - sp preparedModel = nullptr; - saveModelToCache(model, modelCache, dataCache, &preparedModel); - ASSERT_NE(preparedModel, nullptr); - // Execute and verify results. - EvaluatePreparedModel(preparedModel, testModel, - /*testDynamicOutputShape=*/false); - // Check if prepareModelFromCache fails. - preparedModel = nullptr; - ErrorStatus status; - prepareModelFromCache(modelCache, dataCache, &preparedModel, &status); - if (status != ErrorStatus::INVALID_ARGUMENT) { - ASSERT_EQ(status, ErrorStatus::GENERAL_FAILURE); - } - ASSERT_EQ(preparedModel, nullptr); - } - - // Go through each handle in data cache, test with invalid access mode. - for (uint32_t i = 0; i < mNumDataCache; i++) { - hidl_vec modelCache, dataCache; - dataCacheMode[i] = AccessMode::READ_ONLY; - createCacheHandles(mModelCache, modelCacheMode, &modelCache); - createCacheHandles(mDataCache, dataCacheMode, &dataCache); - dataCacheMode[i] = AccessMode::READ_WRITE; - sp preparedModel = nullptr; - saveModelToCache(model, modelCache, dataCache, &preparedModel); - ASSERT_NE(preparedModel, nullptr); - // Execute and verify results. - EvaluatePreparedModel(preparedModel, testModel, - /*testDynamicOutputShape=*/false); - // Check if prepareModelFromCache fails. - preparedModel = nullptr; - ErrorStatus status; - prepareModelFromCache(modelCache, dataCache, &preparedModel, &status); - if (status != ErrorStatus::INVALID_ARGUMENT) { - ASSERT_EQ(status, ErrorStatus::GENERAL_FAILURE); - } - ASSERT_EQ(preparedModel, nullptr); - } -} - -TEST_P(CompilationCachingTest, PrepareModelFromCacheInvalidAccessMode) { - // Create test HIDL model and compile. - const TestModel& testModel = createTestModel(); - const Model model = createModel(testModel); - if (checkEarlyTermination(model)) return; - std::vector modelCacheMode(mNumModelCache, AccessMode::READ_WRITE); - std::vector dataCacheMode(mNumDataCache, AccessMode::READ_WRITE); - - // Save the compilation to cache. - { - hidl_vec modelCache, dataCache; - createCacheHandles(mModelCache, AccessMode::READ_WRITE, &modelCache); - createCacheHandles(mDataCache, AccessMode::READ_WRITE, &dataCache); - saveModelToCache(model, modelCache, dataCache); - } - - // Go through each handle in model cache, test with invalid access mode. - for (uint32_t i = 0; i < mNumModelCache; i++) { - sp preparedModel = nullptr; - ErrorStatus status; - hidl_vec modelCache, dataCache; - modelCacheMode[i] = AccessMode::WRITE_ONLY; - createCacheHandles(mModelCache, modelCacheMode, &modelCache); - createCacheHandles(mDataCache, dataCacheMode, &dataCache); - modelCacheMode[i] = AccessMode::READ_WRITE; - prepareModelFromCache(modelCache, dataCache, &preparedModel, &status); - ASSERT_EQ(status, ErrorStatus::GENERAL_FAILURE); - ASSERT_EQ(preparedModel, nullptr); - } - - // Go through each handle in data cache, test with invalid access mode. - for (uint32_t i = 0; i < mNumDataCache; i++) { - sp preparedModel = nullptr; - ErrorStatus status; - hidl_vec modelCache, dataCache; - dataCacheMode[i] = AccessMode::WRITE_ONLY; - createCacheHandles(mModelCache, modelCacheMode, &modelCache); - createCacheHandles(mDataCache, dataCacheMode, &dataCache); - dataCacheMode[i] = AccessMode::READ_WRITE; - prepareModelFromCache(modelCache, dataCache, &preparedModel, &status); - ASSERT_EQ(status, ErrorStatus::GENERAL_FAILURE); - ASSERT_EQ(preparedModel, nullptr); - } -} - -// Copy file contents between file groups. -// The outer vector corresponds to handles and the inner vector is for fds held by each handle. -// The outer vector sizes must match and the inner vectors must have size = 1. -static void copyCacheFiles(const std::vector>& from, - const std::vector>& to) { - constexpr size_t kBufferSize = 1000000; - uint8_t buffer[kBufferSize]; - - ASSERT_EQ(from.size(), to.size()); - for (uint32_t i = 0; i < from.size(); i++) { - ASSERT_EQ(from[i].size(), 1u); - ASSERT_EQ(to[i].size(), 1u); - int fromFd = open(from[i][0].c_str(), O_RDONLY); - int toFd = open(to[i][0].c_str(), O_WRONLY | O_CREAT, S_IRUSR | S_IWUSR); - ASSERT_GE(fromFd, 0); - ASSERT_GE(toFd, 0); - - ssize_t readBytes; - while ((readBytes = read(fromFd, &buffer, kBufferSize)) > 0) { - ASSERT_EQ(write(toFd, &buffer, readBytes), readBytes); - } - ASSERT_GE(readBytes, 0); - - close(fromFd); - close(toFd); - } -} - -// Number of operations in the large test model. -constexpr uint32_t kLargeModelSize = 100; -constexpr uint32_t kNumIterationsTOCTOU = 100; - -TEST_P(CompilationCachingTest, SaveToCache_TOCTOU) { - if (!mIsCachingSupported) return; - - // Create test models and check if fully supported by the service. - const TestModel testModelMul = createLargeTestModel(OperationType::MUL, kLargeModelSize); - const Model modelMul = createModel(testModelMul); - if (checkEarlyTermination(modelMul)) return; - const TestModel testModelAdd = createLargeTestModel(OperationType::ADD, kLargeModelSize); - const Model modelAdd = createModel(testModelAdd); - if (checkEarlyTermination(modelAdd)) return; - - // Save the modelMul compilation to cache. - auto modelCacheMul = mModelCache; - for (auto& cache : modelCacheMul) { - cache[0].append("_mul"); - } - { - hidl_vec modelCache, dataCache; - createCacheHandles(modelCacheMul, AccessMode::READ_WRITE, &modelCache); - createCacheHandles(mDataCache, AccessMode::READ_WRITE, &dataCache); - saveModelToCache(modelMul, modelCache, dataCache); - } - - // Use a different token for modelAdd. - mToken[0]++; - - // This test is probabilistic, so we run it multiple times. - for (uint32_t i = 0; i < kNumIterationsTOCTOU; i++) { - // Save the modelAdd compilation to cache. - { - hidl_vec modelCache, dataCache; - createCacheHandles(mModelCache, AccessMode::READ_WRITE, &modelCache); - createCacheHandles(mDataCache, AccessMode::READ_WRITE, &dataCache); - - // Spawn a thread to copy the cache content concurrently while saving to cache. - std::thread thread(copyCacheFiles, std::cref(modelCacheMul), std::cref(mModelCache)); - saveModelToCache(modelAdd, modelCache, dataCache); - thread.join(); - } - - // Retrieve preparedModel from cache. - { - sp preparedModel = nullptr; - ErrorStatus status; - hidl_vec modelCache, dataCache; - createCacheHandles(mModelCache, AccessMode::READ_WRITE, &modelCache); - createCacheHandles(mDataCache, AccessMode::READ_WRITE, &dataCache); - prepareModelFromCache(modelCache, dataCache, &preparedModel, &status); - - // The preparation may fail or succeed, but must not crash. If the preparation succeeds, - // the prepared model must be executed with the correct result and not crash. - if (status != ErrorStatus::NONE) { - ASSERT_EQ(preparedModel, nullptr); - } else { - ASSERT_NE(preparedModel, nullptr); - EvaluatePreparedModel(preparedModel, testModelAdd, - /*testDynamicOutputShape=*/false); - } - } - } -} - -TEST_P(CompilationCachingTest, PrepareFromCache_TOCTOU) { - if (!mIsCachingSupported) return; - - // Create test models and check if fully supported by the service. - const TestModel testModelMul = createLargeTestModel(OperationType::MUL, kLargeModelSize); - const Model modelMul = createModel(testModelMul); - if (checkEarlyTermination(modelMul)) return; - const TestModel testModelAdd = createLargeTestModel(OperationType::ADD, kLargeModelSize); - const Model modelAdd = createModel(testModelAdd); - if (checkEarlyTermination(modelAdd)) return; - - // Save the modelMul compilation to cache. - auto modelCacheMul = mModelCache; - for (auto& cache : modelCacheMul) { - cache[0].append("_mul"); - } - { - hidl_vec modelCache, dataCache; - createCacheHandles(modelCacheMul, AccessMode::READ_WRITE, &modelCache); - createCacheHandles(mDataCache, AccessMode::READ_WRITE, &dataCache); - saveModelToCache(modelMul, modelCache, dataCache); - } - - // Use a different token for modelAdd. - mToken[0]++; - - // This test is probabilistic, so we run it multiple times. - for (uint32_t i = 0; i < kNumIterationsTOCTOU; i++) { - // Save the modelAdd compilation to cache. - { - hidl_vec modelCache, dataCache; - createCacheHandles(mModelCache, AccessMode::READ_WRITE, &modelCache); - createCacheHandles(mDataCache, AccessMode::READ_WRITE, &dataCache); - saveModelToCache(modelAdd, modelCache, dataCache); - } - - // Retrieve preparedModel from cache. - { - sp preparedModel = nullptr; - ErrorStatus status; - hidl_vec modelCache, dataCache; - createCacheHandles(mModelCache, AccessMode::READ_WRITE, &modelCache); - createCacheHandles(mDataCache, AccessMode::READ_WRITE, &dataCache); - - // Spawn a thread to copy the cache content concurrently while preparing from cache. - std::thread thread(copyCacheFiles, std::cref(modelCacheMul), std::cref(mModelCache)); - prepareModelFromCache(modelCache, dataCache, &preparedModel, &status); - thread.join(); - - // The preparation may fail or succeed, but must not crash. If the preparation succeeds, - // the prepared model must be executed with the correct result and not crash. - if (status != ErrorStatus::NONE) { - ASSERT_EQ(preparedModel, nullptr); - } else { - ASSERT_NE(preparedModel, nullptr); - EvaluatePreparedModel(preparedModel, testModelAdd, - /*testDynamicOutputShape=*/false); - } - } - } -} - -TEST_P(CompilationCachingTest, ReplaceSecuritySensitiveCache) { - if (!mIsCachingSupported) return; - - // Create test models and check if fully supported by the service. - const TestModel testModelMul = createLargeTestModel(OperationType::MUL, kLargeModelSize); - const Model modelMul = createModel(testModelMul); - if (checkEarlyTermination(modelMul)) return; - const TestModel testModelAdd = createLargeTestModel(OperationType::ADD, kLargeModelSize); - const Model modelAdd = createModel(testModelAdd); - if (checkEarlyTermination(modelAdd)) return; - - // Save the modelMul compilation to cache. - auto modelCacheMul = mModelCache; - for (auto& cache : modelCacheMul) { - cache[0].append("_mul"); - } - { - hidl_vec modelCache, dataCache; - createCacheHandles(modelCacheMul, AccessMode::READ_WRITE, &modelCache); - createCacheHandles(mDataCache, AccessMode::READ_WRITE, &dataCache); - saveModelToCache(modelMul, modelCache, dataCache); - } - - // Use a different token for modelAdd. - mToken[0]++; - - // Save the modelAdd compilation to cache. - { - hidl_vec modelCache, dataCache; - createCacheHandles(mModelCache, AccessMode::READ_WRITE, &modelCache); - createCacheHandles(mDataCache, AccessMode::READ_WRITE, &dataCache); - saveModelToCache(modelAdd, modelCache, dataCache); - } - - // Replace the model cache of modelAdd with modelMul. - copyCacheFiles(modelCacheMul, mModelCache); - - // Retrieve the preparedModel from cache, expect failure. - { - sp preparedModel = nullptr; - ErrorStatus status; - hidl_vec modelCache, dataCache; - createCacheHandles(mModelCache, AccessMode::READ_WRITE, &modelCache); - createCacheHandles(mDataCache, AccessMode::READ_WRITE, &dataCache); - prepareModelFromCache(modelCache, dataCache, &preparedModel, &status); - ASSERT_EQ(status, ErrorStatus::GENERAL_FAILURE); - ASSERT_EQ(preparedModel, nullptr); - } -} - -static const auto kNamedDeviceChoices = testing::ValuesIn(getNamedDevices()); -static const auto kOperandTypeChoices = - testing::Values(OperandType::TENSOR_FLOAT32, OperandType::TENSOR_QUANT8_ASYMM); - -std::string printCompilationCachingTest( - const testing::TestParamInfo& info) { - const auto& [namedDevice, operandType] = info.param; - const std::string type = (operandType == OperandType::TENSOR_FLOAT32 ? "float32" : "quant8"); - return gtestCompliantName(getName(namedDevice) + "_" + type); -} - -INSTANTIATE_TEST_CASE_P(TestCompilationCaching, CompilationCachingTest, - testing::Combine(kNamedDeviceChoices, kOperandTypeChoices), - printCompilationCachingTest); - -using CompilationCachingSecurityTestParam = std::tuple; - -class CompilationCachingSecurityTest - : public CompilationCachingTestBase, - public testing::WithParamInterface { - protected: - CompilationCachingSecurityTest() - : CompilationCachingTestBase(getData(std::get(GetParam())), - std::get(GetParam())) {} - - void SetUp() { - CompilationCachingTestBase::SetUp(); - generator.seed(kSeed); - } - - // Get a random integer within a closed range [lower, upper]. - template - T getRandomInt(T lower, T upper) { - std::uniform_int_distribution dis(lower, upper); - return dis(generator); - } - - // Randomly flip one single bit of the cache entry. - void flipOneBitOfCache(const std::string& filename, bool* skip) { - FILE* pFile = fopen(filename.c_str(), "r+"); - ASSERT_EQ(fseek(pFile, 0, SEEK_END), 0); - long int fileSize = ftell(pFile); - if (fileSize == 0) { - fclose(pFile); - *skip = true; - return; - } - ASSERT_EQ(fseek(pFile, getRandomInt(0l, fileSize - 1), SEEK_SET), 0); - int readByte = fgetc(pFile); - ASSERT_NE(readByte, EOF); - ASSERT_EQ(fseek(pFile, -1, SEEK_CUR), 0); - ASSERT_NE(fputc(static_cast(readByte) ^ (1U << getRandomInt(0, 7)), pFile), EOF); - fclose(pFile); - *skip = false; - } - - // Randomly append bytes to the cache entry. - void appendBytesToCache(const std::string& filename, bool* skip) { - FILE* pFile = fopen(filename.c_str(), "a"); - uint32_t appendLength = getRandomInt(1, 256); - for (uint32_t i = 0; i < appendLength; i++) { - ASSERT_NE(fputc(getRandomInt(0, 255), pFile), EOF); - } - fclose(pFile); - *skip = false; - } - - enum class ExpectedResult { GENERAL_FAILURE, NOT_CRASH }; - - // Test if the driver behaves as expected when given corrupted cache or token. - // The modifier will be invoked after save to cache but before prepare from cache. - // The modifier accepts one pointer argument "skip" as the returning value, indicating - // whether the test should be skipped or not. - void testCorruptedCache(ExpectedResult expected, std::function modifier) { - const TestModel& testModel = createTestModel(); - const Model model = createModel(testModel); - if (checkEarlyTermination(model)) return; - - // Save the compilation to cache. - { - hidl_vec modelCache, dataCache; - createCacheHandles(mModelCache, AccessMode::READ_WRITE, &modelCache); - createCacheHandles(mDataCache, AccessMode::READ_WRITE, &dataCache); - saveModelToCache(model, modelCache, dataCache); - } - - bool skip = false; - modifier(&skip); - if (skip) return; - - // Retrieve preparedModel from cache. - { - sp preparedModel = nullptr; - ErrorStatus status; - hidl_vec modelCache, dataCache; - createCacheHandles(mModelCache, AccessMode::READ_WRITE, &modelCache); - createCacheHandles(mDataCache, AccessMode::READ_WRITE, &dataCache); - prepareModelFromCache(modelCache, dataCache, &preparedModel, &status); - - switch (expected) { - case ExpectedResult::GENERAL_FAILURE: - ASSERT_EQ(status, ErrorStatus::GENERAL_FAILURE); - ASSERT_EQ(preparedModel, nullptr); - break; - case ExpectedResult::NOT_CRASH: - ASSERT_EQ(preparedModel == nullptr, status != ErrorStatus::NONE); - break; - default: - FAIL(); - } - } - } - - const uint32_t kSeed = std::get(GetParam()); - std::mt19937 generator; -}; - -TEST_P(CompilationCachingSecurityTest, CorruptedModelCache) { - if (!mIsCachingSupported) return; - for (uint32_t i = 0; i < mNumModelCache; i++) { - testCorruptedCache(ExpectedResult::GENERAL_FAILURE, - [this, i](bool* skip) { flipOneBitOfCache(mModelCache[i][0], skip); }); - } -} - -TEST_P(CompilationCachingSecurityTest, WrongLengthModelCache) { - if (!mIsCachingSupported) return; - for (uint32_t i = 0; i < mNumModelCache; i++) { - testCorruptedCache(ExpectedResult::GENERAL_FAILURE, - [this, i](bool* skip) { appendBytesToCache(mModelCache[i][0], skip); }); - } -} - -TEST_P(CompilationCachingSecurityTest, CorruptedDataCache) { - if (!mIsCachingSupported) return; - for (uint32_t i = 0; i < mNumDataCache; i++) { - testCorruptedCache(ExpectedResult::NOT_CRASH, - [this, i](bool* skip) { flipOneBitOfCache(mDataCache[i][0], skip); }); - } -} - -TEST_P(CompilationCachingSecurityTest, WrongLengthDataCache) { - if (!mIsCachingSupported) return; - for (uint32_t i = 0; i < mNumDataCache; i++) { - testCorruptedCache(ExpectedResult::NOT_CRASH, - [this, i](bool* skip) { appendBytesToCache(mDataCache[i][0], skip); }); - } -} - -TEST_P(CompilationCachingSecurityTest, WrongToken) { - if (!mIsCachingSupported) return; - testCorruptedCache(ExpectedResult::GENERAL_FAILURE, [this](bool* skip) { - // Randomly flip one single bit in mToken. - uint32_t ind = - getRandomInt(0u, static_cast(Constant::BYTE_SIZE_OF_CACHE_TOKEN) - 1); - mToken[ind] ^= (1U << getRandomInt(0, 7)); - *skip = false; - }); -} - -std::string printCompilationCachingSecurityTest( - const testing::TestParamInfo& info) { - const auto& [namedDevice, operandType, seed] = info.param; - const std::string type = (operandType == OperandType::TENSOR_FLOAT32 ? "float32" : "quant8"); - return gtestCompliantName(getName(namedDevice) + "_" + type + "_" + std::to_string(seed)); -} - -INSTANTIATE_TEST_CASE_P(TestCompilationCaching, CompilationCachingSecurityTest, - testing::Combine(kNamedDeviceChoices, kOperandTypeChoices, - testing::Range(0U, 10U)), - printCompilationCachingSecurityTest); - -} // namespace android::hardware::neuralnetworks::V1_2::vts::functional diff --git a/neuralnetworks/1.3/vts/functional/GeneratedTestHarness.cpp b/neuralnetworks/1.3/vts/functional/GeneratedTestHarness.cpp deleted file mode 100644 index 2beec983e0..0000000000 --- a/neuralnetworks/1.3/vts/functional/GeneratedTestHarness.cpp +++ /dev/null @@ -1,408 +0,0 @@ -/* - * Copyright (C) 2019 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "GeneratedTestHarness.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include - -#include "1.0/Utils.h" -#include "1.2/Callbacks.h" -#include "ExecutionBurstController.h" -#include "MemoryUtils.h" -#include "TestHarness.h" -#include "Utils.h" -#include "VtsHalNeuralnetworks.h" - -namespace android::hardware::neuralnetworks::V1_2::vts::functional { - -using namespace test_helper; -using hidl::memory::V1_0::IMemory; -using implementation::ExecutionCallback; -using implementation::PreparedModelCallback; -using V1_0::DataLocation; -using V1_0::ErrorStatus; -using V1_0::OperandLifeTime; -using V1_0::Request; -using V1_1::ExecutionPreference; -using HidlToken = hidl_array(Constant::BYTE_SIZE_OF_CACHE_TOKEN)>; - -enum class OutputType { FULLY_SPECIFIED, UNSPECIFIED, INSUFFICIENT }; - -Model createModel(const TestModel& testModel) { - // Model operands. - hidl_vec operands(testModel.operands.size()); - size_t constCopySize = 0, constRefSize = 0; - for (uint32_t i = 0; i < testModel.operands.size(); i++) { - const auto& op = testModel.operands[i]; - - DataLocation loc = {}; - if (op.lifetime == TestOperandLifeTime::CONSTANT_COPY) { - loc = {.poolIndex = 0, - .offset = static_cast(constCopySize), - .length = static_cast(op.data.size())}; - constCopySize += op.data.alignedSize(); - } else if (op.lifetime == TestOperandLifeTime::CONSTANT_REFERENCE) { - loc = {.poolIndex = 0, - .offset = static_cast(constRefSize), - .length = static_cast(op.data.size())}; - constRefSize += op.data.alignedSize(); - } - - Operand::ExtraParams extraParams; - if (op.type == TestOperandType::TENSOR_QUANT8_SYMM_PER_CHANNEL) { - extraParams.channelQuant(SymmPerChannelQuantParams{ - .scales = op.channelQuant.scales, .channelDim = op.channelQuant.channelDim}); - } - - operands[i] = {.type = static_cast(op.type), - .dimensions = op.dimensions, - .numberOfConsumers = op.numberOfConsumers, - .scale = op.scale, - .zeroPoint = op.zeroPoint, - .lifetime = static_cast(op.lifetime), - .location = loc, - .extraParams = std::move(extraParams)}; - } - - // Model operations. - hidl_vec operations(testModel.operations.size()); - std::transform(testModel.operations.begin(), testModel.operations.end(), operations.begin(), - [](const TestOperation& op) -> Operation { - return {.type = static_cast(op.type), - .inputs = op.inputs, - .outputs = op.outputs}; - }); - - // Constant copies. - hidl_vec operandValues(constCopySize); - for (uint32_t i = 0; i < testModel.operands.size(); i++) { - const auto& op = testModel.operands[i]; - if (op.lifetime == TestOperandLifeTime::CONSTANT_COPY) { - const uint8_t* begin = op.data.get(); - const uint8_t* end = begin + op.data.size(); - std::copy(begin, end, operandValues.data() + operands[i].location.offset); - } - } - - // Shared memory. - hidl_vec pools = {}; - if (constRefSize > 0) { - hidl_vec_push_back(&pools, nn::allocateSharedMemory(constRefSize)); - CHECK_NE(pools[0].size(), 0u); - - // load data - sp mappedMemory = mapMemory(pools[0]); - CHECK(mappedMemory.get() != nullptr); - uint8_t* mappedPtr = - reinterpret_cast(static_cast(mappedMemory->getPointer())); - CHECK(mappedPtr != nullptr); - - for (uint32_t i = 0; i < testModel.operands.size(); i++) { - const auto& op = testModel.operands[i]; - if (op.lifetime == TestOperandLifeTime::CONSTANT_REFERENCE) { - const uint8_t* begin = op.data.get(); - const uint8_t* end = begin + op.data.size(); - std::copy(begin, end, mappedPtr + operands[i].location.offset); - } - } - } - - return {.operands = std::move(operands), - .operations = std::move(operations), - .inputIndexes = testModel.inputIndexes, - .outputIndexes = testModel.outputIndexes, - .operandValues = std::move(operandValues), - .pools = std::move(pools), - .relaxComputationFloat32toFloat16 = testModel.isRelaxed}; -} - -static bool isOutputSizeGreaterThanOne(const TestModel& testModel, uint32_t index) { - const auto byteSize = testModel.operands[testModel.outputIndexes[index]].data.size(); - return byteSize > 1u; -} - -static void makeOutputInsufficientSize(uint32_t outputIndex, Request* request) { - auto& length = request->outputs[outputIndex].location.length; - ASSERT_GT(length, 1u); - length -= 1u; -} - -static void makeOutputDimensionsUnspecified(Model* model) { - for (auto i : model->outputIndexes) { - auto& dims = model->operands[i].dimensions; - std::fill(dims.begin(), dims.end(), 0); - } -} - -static Return ExecutePreparedModel(const sp& preparedModel, - const Request& request, MeasureTiming measure, - sp& callback) { - return preparedModel->execute_1_2(request, measure, callback); -} -static Return ExecutePreparedModel(const sp& preparedModel, - const Request& request, MeasureTiming measure, - hidl_vec* outputShapes, - Timing* timing) { - ErrorStatus result; - Return ret = preparedModel->executeSynchronously( - request, measure, - [&result, outputShapes, timing](ErrorStatus error, const hidl_vec& shapes, - const Timing& time) { - result = error; - *outputShapes = shapes; - *timing = time; - }); - if (!ret.isOk()) { - return ErrorStatus::GENERAL_FAILURE; - } - return result; -} -static std::shared_ptr<::android::nn::ExecutionBurstController> CreateBurst( - const sp& preparedModel) { - return android::nn::ExecutionBurstController::create(preparedModel, /*blocking=*/true); -} -enum class Executor { ASYNC, SYNC, BURST }; - -void EvaluatePreparedModel(const sp& preparedModel, const TestModel& testModel, - Executor executor, MeasureTiming measure, OutputType outputType) { - // If output0 does not have size larger than one byte, we can not test with insufficient buffer. - if (outputType == OutputType::INSUFFICIENT && !isOutputSizeGreaterThanOne(testModel, 0)) { - return; - } - - Request request = createRequest(testModel); - if (outputType == OutputType::INSUFFICIENT) { - makeOutputInsufficientSize(/*outputIndex=*/0, &request); - } - - ErrorStatus executionStatus; - hidl_vec outputShapes; - Timing timing; - switch (executor) { - case Executor::ASYNC: { - SCOPED_TRACE("asynchronous"); - - // launch execution - sp executionCallback = new ExecutionCallback(); - Return executionLaunchStatus = - ExecutePreparedModel(preparedModel, request, measure, executionCallback); - ASSERT_TRUE(executionLaunchStatus.isOk()); - EXPECT_EQ(ErrorStatus::NONE, static_cast(executionLaunchStatus)); - - // retrieve execution status - executionCallback->wait(); - executionStatus = executionCallback->getStatus(); - outputShapes = executionCallback->getOutputShapes(); - timing = executionCallback->getTiming(); - - break; - } - case Executor::SYNC: { - SCOPED_TRACE("synchronous"); - - // execute - Return executionReturnStatus = - ExecutePreparedModel(preparedModel, request, measure, &outputShapes, &timing); - ASSERT_TRUE(executionReturnStatus.isOk()); - executionStatus = static_cast(executionReturnStatus); - - break; - } - case Executor::BURST: { - SCOPED_TRACE("burst"); - - // create burst - const std::shared_ptr<::android::nn::ExecutionBurstController> controller = - CreateBurst(preparedModel); - ASSERT_NE(nullptr, controller.get()); - - // create memory keys - std::vector keys(request.pools.size()); - for (size_t i = 0; i < keys.size(); ++i) { - keys[i] = reinterpret_cast(&request.pools[i]); - } - - // execute burst - std::tie(executionStatus, outputShapes, timing) = - controller->compute(request, measure, keys); - - break; - } - } - - if (outputType != OutputType::FULLY_SPECIFIED && - executionStatus == ErrorStatus::GENERAL_FAILURE) { - LOG(INFO) << "NN VTS: Early termination of test because vendor service cannot " - "execute model that it does not support."; - std::cout << "[ ] Early termination of test because vendor service cannot " - "execute model that it does not support." - << std::endl; - GTEST_SKIP(); - } - if (measure == MeasureTiming::NO) { - EXPECT_EQ(UINT64_MAX, timing.timeOnDevice); - EXPECT_EQ(UINT64_MAX, timing.timeInDriver); - } else { - if (timing.timeOnDevice != UINT64_MAX && timing.timeInDriver != UINT64_MAX) { - EXPECT_LE(timing.timeOnDevice, timing.timeInDriver); - } - } - - switch (outputType) { - case OutputType::FULLY_SPECIFIED: - // If the model output operands are fully specified, outputShapes must be either - // either empty, or have the same number of elements as the number of outputs. - ASSERT_EQ(ErrorStatus::NONE, executionStatus); - ASSERT_TRUE(outputShapes.size() == 0 || - outputShapes.size() == testModel.outputIndexes.size()); - break; - case OutputType::UNSPECIFIED: - // If the model output operands are not fully specified, outputShapes must have - // the same number of elements as the number of outputs. - ASSERT_EQ(ErrorStatus::NONE, executionStatus); - ASSERT_EQ(outputShapes.size(), testModel.outputIndexes.size()); - break; - case OutputType::INSUFFICIENT: - ASSERT_EQ(ErrorStatus::OUTPUT_INSUFFICIENT_SIZE, executionStatus); - ASSERT_EQ(outputShapes.size(), testModel.outputIndexes.size()); - ASSERT_FALSE(outputShapes[0].isSufficient); - return; - } - - // Go through all outputs, check returned output shapes. - for (uint32_t i = 0; i < outputShapes.size(); i++) { - EXPECT_TRUE(outputShapes[i].isSufficient); - const auto& expect = testModel.operands[testModel.outputIndexes[i]].dimensions; - const std::vector actual = outputShapes[i].dimensions; - EXPECT_EQ(expect, actual); - } - - // Retrieve execution results. - const std::vector outputs = getOutputBuffers(request); - - // We want "close-enough" results. - checkResults(testModel, outputs); -} - -void EvaluatePreparedModel(const sp& preparedModel, const TestModel& testModel, - bool testDynamicOutputShape) { - if (testDynamicOutputShape) { - EvaluatePreparedModel(preparedModel, testModel, Executor::ASYNC, MeasureTiming::NO, - OutputType::UNSPECIFIED); - EvaluatePreparedModel(preparedModel, testModel, Executor::SYNC, MeasureTiming::NO, - OutputType::UNSPECIFIED); - EvaluatePreparedModel(preparedModel, testModel, Executor::BURST, MeasureTiming::NO, - OutputType::UNSPECIFIED); - EvaluatePreparedModel(preparedModel, testModel, Executor::ASYNC, MeasureTiming::YES, - OutputType::UNSPECIFIED); - EvaluatePreparedModel(preparedModel, testModel, Executor::SYNC, MeasureTiming::YES, - OutputType::UNSPECIFIED); - EvaluatePreparedModel(preparedModel, testModel, Executor::BURST, MeasureTiming::YES, - OutputType::UNSPECIFIED); - EvaluatePreparedModel(preparedModel, testModel, Executor::ASYNC, MeasureTiming::NO, - OutputType::INSUFFICIENT); - EvaluatePreparedModel(preparedModel, testModel, Executor::SYNC, MeasureTiming::NO, - OutputType::INSUFFICIENT); - EvaluatePreparedModel(preparedModel, testModel, Executor::BURST, MeasureTiming::NO, - OutputType::INSUFFICIENT); - EvaluatePreparedModel(preparedModel, testModel, Executor::ASYNC, MeasureTiming::YES, - OutputType::INSUFFICIENT); - EvaluatePreparedModel(preparedModel, testModel, Executor::SYNC, MeasureTiming::YES, - OutputType::INSUFFICIENT); - EvaluatePreparedModel(preparedModel, testModel, Executor::BURST, MeasureTiming::YES, - OutputType::INSUFFICIENT); - } else { - EvaluatePreparedModel(preparedModel, testModel, Executor::ASYNC, MeasureTiming::NO, - OutputType::FULLY_SPECIFIED); - EvaluatePreparedModel(preparedModel, testModel, Executor::SYNC, MeasureTiming::NO, - OutputType::FULLY_SPECIFIED); - EvaluatePreparedModel(preparedModel, testModel, Executor::BURST, MeasureTiming::NO, - OutputType::FULLY_SPECIFIED); - EvaluatePreparedModel(preparedModel, testModel, Executor::ASYNC, MeasureTiming::YES, - OutputType::FULLY_SPECIFIED); - EvaluatePreparedModel(preparedModel, testModel, Executor::SYNC, MeasureTiming::YES, - OutputType::FULLY_SPECIFIED); - EvaluatePreparedModel(preparedModel, testModel, Executor::BURST, MeasureTiming::YES, - OutputType::FULLY_SPECIFIED); - } -} - -void Execute(const sp& device, const TestModel& testModel, bool testDynamicOutputShape) { - Model model = createModel(testModel); - if (testDynamicOutputShape) { - makeOutputDimensionsUnspecified(&model); - } - - sp preparedModel; - createPreparedModel(device, model, &preparedModel); - if (preparedModel == nullptr) return; - - EvaluatePreparedModel(preparedModel, testModel, testDynamicOutputShape); -} - -void GeneratedTestBase::SetUp() { - testing::TestWithParam::SetUp(); - ASSERT_NE(kDevice, nullptr); -} - -std::vector getNamedModels(const FilterFn& filter) { - return TestModelManager::get().getTestModels(filter); -} - -std::string printGeneratedTest(const testing::TestParamInfo& info) { - const auto& [namedDevice, namedModel] = info.param; - return gtestCompliantName(getName(namedDevice) + "_" + getName(namedModel)); -} - -// Tag for the generated tests -class GeneratedTest : public GeneratedTestBase {}; - -// Tag for the dynamic output shape tests -class DynamicOutputShapeTest : public GeneratedTest {}; - -TEST_P(GeneratedTest, Test) { - Execute(kDevice, kTestModel, /*testDynamicOutputShape=*/false); -} - -TEST_P(DynamicOutputShapeTest, Test) { - Execute(kDevice, kTestModel, /*testDynamicOutputShape=*/true); -} - -INSTANTIATE_GENERATED_TEST(GeneratedTest, - [](const TestModel& testModel) { return !testModel.expectFailure; }); - -INSTANTIATE_GENERATED_TEST(DynamicOutputShapeTest, - [](const TestModel& testModel) { return !testModel.expectFailure; }); - -} // namespace android::hardware::neuralnetworks::V1_2::vts::functional diff --git a/neuralnetworks/1.3/vts/functional/GeneratedTestHarness.h b/neuralnetworks/1.3/vts/functional/GeneratedTestHarness.h deleted file mode 100644 index dfc980c169..0000000000 --- a/neuralnetworks/1.3/vts/functional/GeneratedTestHarness.h +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Copyright (C) 2019 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef ANDROID_HARDWARE_NEURALNETWORKS_V1_2_GENERATED_TEST_HARNESS_H -#define ANDROID_HARDWARE_NEURALNETWORKS_V1_2_GENERATED_TEST_HARNESS_H - -#include -#include -#include -#include -#include -#include "1.0/Utils.h" -#include "TestHarness.h" -#include "VtsHalNeuralnetworks.h" - -namespace android::hardware::neuralnetworks::V1_2::vts::functional { - -using NamedModel = Named; -using GeneratedTestParam = std::tuple; - -class GeneratedTestBase : public testing::TestWithParam { - protected: - void SetUp() override; - const sp kDevice = getData(std::get(GetParam())); - const test_helper::TestModel& kTestModel = *getData(std::get(GetParam())); -}; - -using FilterFn = std::function; -std::vector getNamedModels(const FilterFn& filter); - -std::string printGeneratedTest(const testing::TestParamInfo& info); - -#define INSTANTIATE_GENERATED_TEST(TestSuite, filter) \ - INSTANTIATE_TEST_SUITE_P(TestGenerated, TestSuite, \ - testing::Combine(testing::ValuesIn(getNamedDevices()), \ - testing::ValuesIn(getNamedModels(filter))), \ - printGeneratedTest) - -// Tag for the validation tests, instantiated in VtsHalNeuralnetworks.cpp. -// TODO: Clean up the hierarchy for ValidationTest. -class ValidationTest : public GeneratedTestBase {}; - -Model createModel(const test_helper::TestModel& testModel); - -void PrepareModel(const sp& device, const Model& model, sp* preparedModel); - -void EvaluatePreparedModel(const sp& preparedModel, - const test_helper::TestModel& testModel, bool testDynamicOutputShape); - -} // namespace android::hardware::neuralnetworks::V1_2::vts::functional - -#endif // ANDROID_HARDWARE_NEURALNETWORKS_V1_2_GENERATED_TEST_HARNESS_H diff --git a/neuralnetworks/1.3/vts/functional/TestAssertions.cpp b/neuralnetworks/1.3/vts/functional/TestAssertions.cpp deleted file mode 100644 index a0aa3c37d1..0000000000 --- a/neuralnetworks/1.3/vts/functional/TestAssertions.cpp +++ /dev/null @@ -1,141 +0,0 @@ -/* - * Copyright (C) 2019 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include "TestHarness.h" - -namespace android::hardware::neuralnetworks::V1_2 { - -// Make sure that the HIDL enums are compatible with the values defined in -// frameworks/ml/nn/tools/test_generator/test_harness/include/TestHarness.h. -using namespace test_helper; -#define CHECK_TEST_ENUM(EnumType, enumValue) \ - static_assert(static_cast(Test##EnumType::enumValue) == EnumType::enumValue) - -CHECK_TEST_ENUM(OperandType, FLOAT32); -CHECK_TEST_ENUM(OperandType, INT32); -CHECK_TEST_ENUM(OperandType, UINT32); -CHECK_TEST_ENUM(OperandType, TENSOR_FLOAT32); -CHECK_TEST_ENUM(OperandType, TENSOR_INT32); -CHECK_TEST_ENUM(OperandType, TENSOR_QUANT8_ASYMM); -CHECK_TEST_ENUM(OperandType, BOOL); -CHECK_TEST_ENUM(OperandType, TENSOR_QUANT16_SYMM); -CHECK_TEST_ENUM(OperandType, TENSOR_FLOAT16); -CHECK_TEST_ENUM(OperandType, TENSOR_BOOL8); -CHECK_TEST_ENUM(OperandType, FLOAT16); -CHECK_TEST_ENUM(OperandType, TENSOR_QUANT8_SYMM_PER_CHANNEL); -CHECK_TEST_ENUM(OperandType, TENSOR_QUANT16_ASYMM); -CHECK_TEST_ENUM(OperandType, TENSOR_QUANT8_SYMM); - -CHECK_TEST_ENUM(OperationType, ADD); -CHECK_TEST_ENUM(OperationType, AVERAGE_POOL_2D); -CHECK_TEST_ENUM(OperationType, CONCATENATION); -CHECK_TEST_ENUM(OperationType, CONV_2D); -CHECK_TEST_ENUM(OperationType, DEPTHWISE_CONV_2D); -CHECK_TEST_ENUM(OperationType, DEPTH_TO_SPACE); -CHECK_TEST_ENUM(OperationType, DEQUANTIZE); -CHECK_TEST_ENUM(OperationType, EMBEDDING_LOOKUP); -CHECK_TEST_ENUM(OperationType, FLOOR); -CHECK_TEST_ENUM(OperationType, FULLY_CONNECTED); -CHECK_TEST_ENUM(OperationType, HASHTABLE_LOOKUP); -CHECK_TEST_ENUM(OperationType, L2_NORMALIZATION); -CHECK_TEST_ENUM(OperationType, L2_POOL_2D); -CHECK_TEST_ENUM(OperationType, LOCAL_RESPONSE_NORMALIZATION); -CHECK_TEST_ENUM(OperationType, LOGISTIC); -CHECK_TEST_ENUM(OperationType, LSH_PROJECTION); -CHECK_TEST_ENUM(OperationType, LSTM); -CHECK_TEST_ENUM(OperationType, MAX_POOL_2D); -CHECK_TEST_ENUM(OperationType, MUL); -CHECK_TEST_ENUM(OperationType, RELU); -CHECK_TEST_ENUM(OperationType, RELU1); -CHECK_TEST_ENUM(OperationType, RELU6); -CHECK_TEST_ENUM(OperationType, RESHAPE); -CHECK_TEST_ENUM(OperationType, RESIZE_BILINEAR); -CHECK_TEST_ENUM(OperationType, RNN); -CHECK_TEST_ENUM(OperationType, SOFTMAX); -CHECK_TEST_ENUM(OperationType, SPACE_TO_DEPTH); -CHECK_TEST_ENUM(OperationType, SVDF); -CHECK_TEST_ENUM(OperationType, TANH); -CHECK_TEST_ENUM(OperationType, BATCH_TO_SPACE_ND); -CHECK_TEST_ENUM(OperationType, DIV); -CHECK_TEST_ENUM(OperationType, MEAN); -CHECK_TEST_ENUM(OperationType, PAD); -CHECK_TEST_ENUM(OperationType, SPACE_TO_BATCH_ND); -CHECK_TEST_ENUM(OperationType, SQUEEZE); -CHECK_TEST_ENUM(OperationType, STRIDED_SLICE); -CHECK_TEST_ENUM(OperationType, SUB); -CHECK_TEST_ENUM(OperationType, TRANSPOSE); -CHECK_TEST_ENUM(OperationType, ABS); -CHECK_TEST_ENUM(OperationType, ARGMAX); -CHECK_TEST_ENUM(OperationType, ARGMIN); -CHECK_TEST_ENUM(OperationType, AXIS_ALIGNED_BBOX_TRANSFORM); -CHECK_TEST_ENUM(OperationType, BIDIRECTIONAL_SEQUENCE_LSTM); -CHECK_TEST_ENUM(OperationType, BIDIRECTIONAL_SEQUENCE_RNN); -CHECK_TEST_ENUM(OperationType, BOX_WITH_NMS_LIMIT); -CHECK_TEST_ENUM(OperationType, CAST); -CHECK_TEST_ENUM(OperationType, CHANNEL_SHUFFLE); -CHECK_TEST_ENUM(OperationType, DETECTION_POSTPROCESSING); -CHECK_TEST_ENUM(OperationType, EQUAL); -CHECK_TEST_ENUM(OperationType, EXP); -CHECK_TEST_ENUM(OperationType, EXPAND_DIMS); -CHECK_TEST_ENUM(OperationType, GATHER); -CHECK_TEST_ENUM(OperationType, GENERATE_PROPOSALS); -CHECK_TEST_ENUM(OperationType, GREATER); -CHECK_TEST_ENUM(OperationType, GREATER_EQUAL); -CHECK_TEST_ENUM(OperationType, GROUPED_CONV_2D); -CHECK_TEST_ENUM(OperationType, HEATMAP_MAX_KEYPOINT); -CHECK_TEST_ENUM(OperationType, INSTANCE_NORMALIZATION); -CHECK_TEST_ENUM(OperationType, LESS); -CHECK_TEST_ENUM(OperationType, LESS_EQUAL); -CHECK_TEST_ENUM(OperationType, LOG); -CHECK_TEST_ENUM(OperationType, LOGICAL_AND); -CHECK_TEST_ENUM(OperationType, LOGICAL_NOT); -CHECK_TEST_ENUM(OperationType, LOGICAL_OR); -CHECK_TEST_ENUM(OperationType, LOG_SOFTMAX); -CHECK_TEST_ENUM(OperationType, MAXIMUM); -CHECK_TEST_ENUM(OperationType, MINIMUM); -CHECK_TEST_ENUM(OperationType, NEG); -CHECK_TEST_ENUM(OperationType, NOT_EQUAL); -CHECK_TEST_ENUM(OperationType, PAD_V2); -CHECK_TEST_ENUM(OperationType, POW); -CHECK_TEST_ENUM(OperationType, PRELU); -CHECK_TEST_ENUM(OperationType, QUANTIZE); -CHECK_TEST_ENUM(OperationType, QUANTIZED_16BIT_LSTM); -CHECK_TEST_ENUM(OperationType, RANDOM_MULTINOMIAL); -CHECK_TEST_ENUM(OperationType, REDUCE_ALL); -CHECK_TEST_ENUM(OperationType, REDUCE_ANY); -CHECK_TEST_ENUM(OperationType, REDUCE_MAX); -CHECK_TEST_ENUM(OperationType, REDUCE_MIN); -CHECK_TEST_ENUM(OperationType, REDUCE_PROD); -CHECK_TEST_ENUM(OperationType, REDUCE_SUM); -CHECK_TEST_ENUM(OperationType, ROI_ALIGN); -CHECK_TEST_ENUM(OperationType, ROI_POOLING); -CHECK_TEST_ENUM(OperationType, RSQRT); -CHECK_TEST_ENUM(OperationType, SELECT); -CHECK_TEST_ENUM(OperationType, SIN); -CHECK_TEST_ENUM(OperationType, SLICE); -CHECK_TEST_ENUM(OperationType, SPLIT); -CHECK_TEST_ENUM(OperationType, SQRT); -CHECK_TEST_ENUM(OperationType, TILE); -CHECK_TEST_ENUM(OperationType, TOPK_V2); -CHECK_TEST_ENUM(OperationType, TRANSPOSE_CONV_2D); -CHECK_TEST_ENUM(OperationType, UNIDIRECTIONAL_SEQUENCE_LSTM); -CHECK_TEST_ENUM(OperationType, UNIDIRECTIONAL_SEQUENCE_RNN); -CHECK_TEST_ENUM(OperationType, RESIZE_NEAREST_NEIGHBOR); - -#undef CHECK_TEST_ENUM - -} // namespace android::hardware::neuralnetworks::V1_2 diff --git a/neuralnetworks/1.3/vts/functional/ValidateBurst.cpp b/neuralnetworks/1.3/vts/functional/ValidateBurst.cpp deleted file mode 100644 index 1d4493d208..0000000000 --- a/neuralnetworks/1.3/vts/functional/ValidateBurst.cpp +++ /dev/null @@ -1,400 +0,0 @@ -/* - * Copyright (C) 2019 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#define LOG_TAG "neuralnetworks_hidl_hal_test" - -#include "VtsHalNeuralnetworks.h" - -#include "1.2/Callbacks.h" -#include "ExecutionBurstController.h" -#include "ExecutionBurstServer.h" -#include "GeneratedTestHarness.h" -#include "TestHarness.h" -#include "Utils.h" - -#include -#include - -namespace android::hardware::neuralnetworks::V1_2::vts::functional { - -using nn::ExecutionBurstController; -using nn::RequestChannelSender; -using nn::ResultChannelReceiver; -using V1_0::ErrorStatus; -using V1_0::Request; -using ExecutionBurstCallback = ExecutionBurstController::ExecutionBurstCallback; - -// This constant value represents the length of an FMQ that is large enough to -// return a result from a burst execution for all of the generated test cases. -constexpr size_t kExecutionBurstChannelLength = 1024; - -// This constant value represents a length of an FMQ that is not large enough -// to return a result from a burst execution for some of the generated test -// cases. -constexpr size_t kExecutionBurstChannelSmallLength = 8; - -///////////////////////// UTILITY FUNCTIONS ///////////////////////// - -static bool badTiming(Timing timing) { - return timing.timeOnDevice == UINT64_MAX && timing.timeInDriver == UINT64_MAX; -} - -static void createBurst(const sp& preparedModel, const sp& callback, - std::unique_ptr* sender, - std::unique_ptr* receiver, - sp* context, - size_t resultChannelLength = kExecutionBurstChannelLength) { - ASSERT_NE(nullptr, preparedModel.get()); - ASSERT_NE(nullptr, sender); - ASSERT_NE(nullptr, receiver); - ASSERT_NE(nullptr, context); - - // create FMQ objects - auto [fmqRequestChannel, fmqRequestDescriptor] = - RequestChannelSender::create(kExecutionBurstChannelLength, /*blocking=*/true); - auto [fmqResultChannel, fmqResultDescriptor] = - ResultChannelReceiver::create(resultChannelLength, /*blocking=*/true); - ASSERT_NE(nullptr, fmqRequestChannel.get()); - ASSERT_NE(nullptr, fmqResultChannel.get()); - ASSERT_NE(nullptr, fmqRequestDescriptor); - ASSERT_NE(nullptr, fmqResultDescriptor); - - // configure burst - ErrorStatus errorStatus; - sp burstContext; - const Return ret = preparedModel->configureExecutionBurst( - callback, *fmqRequestDescriptor, *fmqResultDescriptor, - [&errorStatus, &burstContext](ErrorStatus status, const sp& context) { - errorStatus = status; - burstContext = context; - }); - ASSERT_TRUE(ret.isOk()); - ASSERT_EQ(ErrorStatus::NONE, errorStatus); - ASSERT_NE(nullptr, burstContext.get()); - - // return values - *sender = std::move(fmqRequestChannel); - *receiver = std::move(fmqResultChannel); - *context = burstContext; -} - -static void createBurstWithResultChannelLength( - const sp& preparedModel, size_t resultChannelLength, - std::shared_ptr* controller) { - ASSERT_NE(nullptr, preparedModel.get()); - ASSERT_NE(nullptr, controller); - - // create FMQ objects - std::unique_ptr sender; - std::unique_ptr receiver; - sp callback = new ExecutionBurstCallback(); - sp context; - ASSERT_NO_FATAL_FAILURE(createBurst(preparedModel, callback, &sender, &receiver, &context, - resultChannelLength)); - ASSERT_NE(nullptr, sender.get()); - ASSERT_NE(nullptr, receiver.get()); - ASSERT_NE(nullptr, context.get()); - - // return values - *controller = std::make_shared(std::move(sender), std::move(receiver), - context, callback); -} - -// Primary validation function. This function will take a valid serialized -// request, apply a mutation to it to invalidate the serialized request, then -// pass it to interface calls that use the serialized request. Note that the -// serialized request here is passed by value, and any mutation to the -// serialized request does not leave this function. -static void validate(RequestChannelSender* sender, ResultChannelReceiver* receiver, - const std::string& message, std::vector serialized, - const std::function*)>& mutation) { - mutation(&serialized); - - // skip if packet is too large to send - if (serialized.size() > kExecutionBurstChannelLength) { - return; - } - - SCOPED_TRACE(message); - - // send invalid packet - ASSERT_TRUE(sender->sendPacket(serialized)); - - // receive error - auto results = receiver->getBlocking(); - ASSERT_TRUE(results.has_value()); - const auto [status, outputShapes, timing] = std::move(*results); - EXPECT_NE(ErrorStatus::NONE, status); - EXPECT_EQ(0u, outputShapes.size()); - EXPECT_TRUE(badTiming(timing)); -} - -// For validation, valid packet entries are mutated to invalid packet entries, -// or invalid packet entries are inserted into valid packets. This function -// creates pre-set invalid packet entries for convenience. -static std::vector createBadRequestPacketEntries() { - const FmqRequestDatum::PacketInformation packetInformation = { - /*.packetSize=*/10, /*.numberOfInputOperands=*/10, /*.numberOfOutputOperands=*/10, - /*.numberOfPools=*/10}; - const FmqRequestDatum::OperandInformation operandInformation = { - /*.hasNoValue=*/false, /*.location=*/{}, /*.numberOfDimensions=*/10}; - const int32_t invalidPoolIdentifier = std::numeric_limits::max(); - std::vector bad(7); - bad[0].packetInformation(packetInformation); - bad[1].inputOperandInformation(operandInformation); - bad[2].inputOperandDimensionValue(0); - bad[3].outputOperandInformation(operandInformation); - bad[4].outputOperandDimensionValue(0); - bad[5].poolIdentifier(invalidPoolIdentifier); - bad[6].measureTiming(MeasureTiming::YES); - return bad; -} - -// For validation, valid packet entries are mutated to invalid packet entries, -// or invalid packet entries are inserted into valid packets. This function -// retrieves pre-set invalid packet entries for convenience. This function -// caches these data so they can be reused on subsequent validation checks. -static const std::vector& getBadRequestPacketEntries() { - static const std::vector bad = createBadRequestPacketEntries(); - return bad; -} - -///////////////////////// REMOVE DATUM //////////////////////////////////// - -static void removeDatumTest(RequestChannelSender* sender, ResultChannelReceiver* receiver, - const std::vector& serialized) { - for (size_t index = 0; index < serialized.size(); ++index) { - const std::string message = "removeDatum: removed datum at index " + std::to_string(index); - validate(sender, receiver, message, serialized, - [index](std::vector* serialized) { - serialized->erase(serialized->begin() + index); - }); - } -} - -///////////////////////// ADD DATUM //////////////////////////////////// - -static void addDatumTest(RequestChannelSender* sender, ResultChannelReceiver* receiver, - const std::vector& serialized) { - const std::vector& extra = getBadRequestPacketEntries(); - for (size_t index = 0; index <= serialized.size(); ++index) { - for (size_t type = 0; type < extra.size(); ++type) { - const std::string message = "addDatum: added datum type " + std::to_string(type) + - " at index " + std::to_string(index); - validate(sender, receiver, message, serialized, - [index, type, &extra](std::vector* serialized) { - serialized->insert(serialized->begin() + index, extra[type]); - }); - } - } -} - -///////////////////////// MUTATE DATUM //////////////////////////////////// - -static bool interestingCase(const FmqRequestDatum& lhs, const FmqRequestDatum& rhs) { - using Discriminator = FmqRequestDatum::hidl_discriminator; - - const bool differentValues = (lhs != rhs); - const bool sameDiscriminator = (lhs.getDiscriminator() == rhs.getDiscriminator()); - const auto discriminator = rhs.getDiscriminator(); - const bool isDimensionValue = (discriminator == Discriminator::inputOperandDimensionValue || - discriminator == Discriminator::outputOperandDimensionValue); - - return differentValues && !(sameDiscriminator && isDimensionValue); -} - -static void mutateDatumTest(RequestChannelSender* sender, ResultChannelReceiver* receiver, - const std::vector& serialized) { - const std::vector& change = getBadRequestPacketEntries(); - for (size_t index = 0; index < serialized.size(); ++index) { - for (size_t type = 0; type < change.size(); ++type) { - if (interestingCase(serialized[index], change[type])) { - const std::string message = "mutateDatum: changed datum at index " + - std::to_string(index) + " to datum type " + - std::to_string(type); - validate(sender, receiver, message, serialized, - [index, type, &change](std::vector* serialized) { - (*serialized)[index] = change[type]; - }); - } - } - } -} - -///////////////////////// BURST VALIATION TESTS //////////////////////////////////// - -static void validateBurstSerialization(const sp& preparedModel, - const Request& request) { - // create burst - std::unique_ptr sender; - std::unique_ptr receiver; - sp callback = new ExecutionBurstCallback(); - sp context; - ASSERT_NO_FATAL_FAILURE(createBurst(preparedModel, callback, &sender, &receiver, &context)); - ASSERT_NE(nullptr, sender.get()); - ASSERT_NE(nullptr, receiver.get()); - ASSERT_NE(nullptr, context.get()); - - // load memory into callback slots - std::vector keys; - keys.reserve(request.pools.size()); - std::transform(request.pools.begin(), request.pools.end(), std::back_inserter(keys), - [](const auto& pool) { return reinterpret_cast(&pool); }); - const std::vector slots = callback->getSlots(request.pools, keys); - - // ensure slot std::numeric_limits::max() doesn't exist (for - // subsequent slot validation testing) - ASSERT_TRUE(std::all_of(slots.begin(), slots.end(), [](int32_t slot) { - return slot != std::numeric_limits::max(); - })); - - // serialize the request - const auto serialized = android::nn::serialize(request, MeasureTiming::YES, slots); - - // validations - removeDatumTest(sender.get(), receiver.get(), serialized); - addDatumTest(sender.get(), receiver.get(), serialized); - mutateDatumTest(sender.get(), receiver.get(), serialized); -} - -// This test validates that when the Result message size exceeds length of the -// result FMQ, the service instance gracefully fails and returns an error. -static void validateBurstFmqLength(const sp& preparedModel, - const Request& request) { - // create regular burst - std::shared_ptr controllerRegular; - ASSERT_NO_FATAL_FAILURE(createBurstWithResultChannelLength( - preparedModel, kExecutionBurstChannelLength, &controllerRegular)); - ASSERT_NE(nullptr, controllerRegular.get()); - - // create burst with small output channel - std::shared_ptr controllerSmall; - ASSERT_NO_FATAL_FAILURE(createBurstWithResultChannelLength( - preparedModel, kExecutionBurstChannelSmallLength, &controllerSmall)); - ASSERT_NE(nullptr, controllerSmall.get()); - - // load memory into callback slots - std::vector keys(request.pools.size()); - for (size_t i = 0; i < keys.size(); ++i) { - keys[i] = reinterpret_cast(&request.pools[i]); - } - - // collect serialized result by running regular burst - const auto [statusRegular, outputShapesRegular, timingRegular] = - controllerRegular->compute(request, MeasureTiming::NO, keys); - - // skip test if regular burst output isn't useful for testing a failure - // caused by having too small of a length for the result FMQ - const std::vector serialized = - android::nn::serialize(statusRegular, outputShapesRegular, timingRegular); - if (statusRegular != ErrorStatus::NONE || - serialized.size() <= kExecutionBurstChannelSmallLength) { - return; - } - - // by this point, execution should fail because the result channel isn't - // large enough to return the serialized result - const auto [statusSmall, outputShapesSmall, timingSmall] = - controllerSmall->compute(request, MeasureTiming::NO, keys); - EXPECT_NE(ErrorStatus::NONE, statusSmall); - EXPECT_EQ(0u, outputShapesSmall.size()); - EXPECT_TRUE(badTiming(timingSmall)); -} - -static bool isSanitized(const FmqResultDatum& datum) { - using Discriminator = FmqResultDatum::hidl_discriminator; - - // check to ensure the padding values in the returned - // FmqResultDatum::OperandInformation are initialized to 0 - if (datum.getDiscriminator() == Discriminator::operandInformation) { - static_assert( - offsetof(FmqResultDatum::OperandInformation, isSufficient) == 0, - "unexpected value for offset of FmqResultDatum::OperandInformation::isSufficient"); - static_assert( - sizeof(FmqResultDatum::OperandInformation::isSufficient) == 1, - "unexpected value for size of FmqResultDatum::OperandInformation::isSufficient"); - static_assert(offsetof(FmqResultDatum::OperandInformation, numberOfDimensions) == 4, - "unexpected value for offset of " - "FmqResultDatum::OperandInformation::numberOfDimensions"); - static_assert(sizeof(FmqResultDatum::OperandInformation::numberOfDimensions) == 4, - "unexpected value for size of " - "FmqResultDatum::OperandInformation::numberOfDimensions"); - static_assert(sizeof(FmqResultDatum::OperandInformation) == 8, - "unexpected value for size of " - "FmqResultDatum::OperandInformation"); - - constexpr size_t paddingOffset = - offsetof(FmqResultDatum::OperandInformation, isSufficient) + - sizeof(FmqResultDatum::OperandInformation::isSufficient); - constexpr size_t paddingSize = - offsetof(FmqResultDatum::OperandInformation, numberOfDimensions) - paddingOffset; - - FmqResultDatum::OperandInformation initialized{}; - std::memset(&initialized, 0, sizeof(initialized)); - - const char* initializedPaddingStart = - reinterpret_cast(&initialized) + paddingOffset; - const char* datumPaddingStart = - reinterpret_cast(&datum.operandInformation()) + paddingOffset; - - return std::memcmp(datumPaddingStart, initializedPaddingStart, paddingSize) == 0; - } - - // there are no other padding initialization checks required, so return true - // for any sum-type that isn't FmqResultDatum::OperandInformation - return true; -} - -static void validateBurstSanitized(const sp& preparedModel, - const Request& request) { - // create burst - std::unique_ptr sender; - std::unique_ptr receiver; - sp callback = new ExecutionBurstCallback(); - sp context; - ASSERT_NO_FATAL_FAILURE(createBurst(preparedModel, callback, &sender, &receiver, &context)); - ASSERT_NE(nullptr, sender.get()); - ASSERT_NE(nullptr, receiver.get()); - ASSERT_NE(nullptr, context.get()); - - // load memory into callback slots - std::vector keys; - keys.reserve(request.pools.size()); - std::transform(request.pools.begin(), request.pools.end(), std::back_inserter(keys), - [](const auto& pool) { return reinterpret_cast(&pool); }); - const std::vector slots = callback->getSlots(request.pools, keys); - - // send valid request - ASSERT_TRUE(sender->send(request, MeasureTiming::YES, slots)); - - // receive valid result - auto serialized = receiver->getPacketBlocking(); - ASSERT_TRUE(serialized.has_value()); - - // sanitize result - ASSERT_TRUE(std::all_of(serialized->begin(), serialized->end(), isSanitized)) - << "The result serialized data is not properly sanitized"; -} - -///////////////////////////// ENTRY POINT ////////////////////////////////// - -void validateBurst(const sp& preparedModel, const Request& request) { - ASSERT_NO_FATAL_FAILURE(validateBurstSerialization(preparedModel, request)); - ASSERT_NO_FATAL_FAILURE(validateBurstFmqLength(preparedModel, request)); - ASSERT_NO_FATAL_FAILURE(validateBurstSanitized(preparedModel, request)); -} - -} // namespace android::hardware::neuralnetworks::V1_2::vts::functional diff --git a/neuralnetworks/1.3/vts/functional/ValidateModel.cpp b/neuralnetworks/1.3/vts/functional/ValidateModel.cpp deleted file mode 100644 index 30530beacc..0000000000 --- a/neuralnetworks/1.3/vts/functional/ValidateModel.cpp +++ /dev/null @@ -1,713 +0,0 @@ -/* - * Copyright (C) 2018 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#define LOG_TAG "neuralnetworks_hidl_hal_test" - -#include "1.0/Utils.h" -#include "1.2/Callbacks.h" -#include "GeneratedTestHarness.h" -#include "VtsHalNeuralnetworks.h" - -namespace android::hardware::neuralnetworks::V1_2::vts::functional { - -using implementation::PreparedModelCallback; -using V1_0::ErrorStatus; -using V1_0::OperandLifeTime; -using V1_1::ExecutionPreference; -using HidlToken = hidl_array(Constant::BYTE_SIZE_OF_CACHE_TOKEN)>; - -///////////////////////// UTILITY FUNCTIONS ///////////////////////// - -static void validateGetSupportedOperations(const sp& device, const std::string& message, - const Model& model) { - SCOPED_TRACE(message + " [getSupportedOperations_1_2]"); - - Return ret = device->getSupportedOperations_1_2( - model, [&](ErrorStatus status, const hidl_vec&) { - EXPECT_EQ(ErrorStatus::INVALID_ARGUMENT, status); - }); - EXPECT_TRUE(ret.isOk()); -} - -static void validatePrepareModel(const sp& device, const std::string& message, - const Model& model, ExecutionPreference preference) { - SCOPED_TRACE(message + " [prepareModel_1_2]"); - - sp preparedModelCallback = new PreparedModelCallback(); - Return prepareLaunchStatus = - device->prepareModel_1_2(model, preference, hidl_vec(), - hidl_vec(), HidlToken(), preparedModelCallback); - ASSERT_TRUE(prepareLaunchStatus.isOk()); - ASSERT_EQ(ErrorStatus::INVALID_ARGUMENT, static_cast(prepareLaunchStatus)); - - preparedModelCallback->wait(); - ErrorStatus prepareReturnStatus = preparedModelCallback->getStatus(); - ASSERT_EQ(ErrorStatus::INVALID_ARGUMENT, prepareReturnStatus); - sp preparedModel = getPreparedModel_1_2(preparedModelCallback); - ASSERT_EQ(nullptr, preparedModel.get()); -} - -static bool validExecutionPreference(ExecutionPreference preference) { - return preference == ExecutionPreference::LOW_POWER || - preference == ExecutionPreference::FAST_SINGLE_ANSWER || - preference == ExecutionPreference::SUSTAINED_SPEED; -} - -// Primary validation function. This function will take a valid model, apply a -// mutation to it to invalidate the model, then pass it to interface calls that -// use the model. Note that the model here is passed by value, and any mutation -// to the model does not leave this function. -static void validate(const sp& device, const std::string& message, Model model, - const std::function& mutation, - ExecutionPreference preference = ExecutionPreference::FAST_SINGLE_ANSWER) { - mutation(&model); - if (validExecutionPreference(preference)) { - validateGetSupportedOperations(device, message, model); - } - validatePrepareModel(device, message, model, preference); -} - -static uint32_t addOperand(Model* model) { - return hidl_vec_push_back(&model->operands, - { - .type = OperandType::INT32, - .dimensions = {}, - .numberOfConsumers = 0, - .scale = 0.0f, - .zeroPoint = 0, - .lifetime = OperandLifeTime::MODEL_INPUT, - .location = {.poolIndex = 0, .offset = 0, .length = 0}, - }); -} - -static uint32_t addOperand(Model* model, OperandLifeTime lifetime) { - uint32_t index = addOperand(model); - model->operands[index].numberOfConsumers = 1; - model->operands[index].lifetime = lifetime; - return index; -} - -///////////////////////// VALIDATE MODEL OPERAND TYPE ///////////////////////// - -static const uint32_t invalidOperandTypes[] = { - static_cast(OperandTypeRange::FUNDAMENTAL_MIN) - 1, - static_cast(OperandTypeRange::FUNDAMENTAL_MAX) + 1, - static_cast(OperandTypeRange::OEM_MIN) - 1, - static_cast(OperandTypeRange::OEM_MAX) + 1, -}; - -static void mutateOperandTypeTest(const sp& device, const Model& model) { - for (size_t operand = 0; operand < model.operands.size(); ++operand) { - for (uint32_t invalidOperandType : invalidOperandTypes) { - const std::string message = "mutateOperandTypeTest: operand " + - std::to_string(operand) + " set to value " + - std::to_string(invalidOperandType); - validate(device, message, model, [operand, invalidOperandType](Model* model) { - model->operands[operand].type = static_cast(invalidOperandType); - }); - } - } -} - -///////////////////////// VALIDATE OPERAND RANK ///////////////////////// - -static uint32_t getInvalidRank(OperandType type) { - switch (type) { - case OperandType::FLOAT16: - case OperandType::FLOAT32: - case OperandType::INT32: - case OperandType::UINT32: - case OperandType::BOOL: - return 1; - case OperandType::TENSOR_BOOL8: - case OperandType::TENSOR_FLOAT16: - case OperandType::TENSOR_FLOAT32: - case OperandType::TENSOR_INT32: - case OperandType::TENSOR_QUANT8_ASYMM: - case OperandType::TENSOR_QUANT8_SYMM: - case OperandType::TENSOR_QUANT16_ASYMM: - case OperandType::TENSOR_QUANT16_SYMM: - case OperandType::TENSOR_QUANT8_SYMM_PER_CHANNEL: - return 0; - default: - return 0; - } -} - -static void mutateOperandRankTest(const sp& device, const Model& model) { - for (size_t operand = 0; operand < model.operands.size(); ++operand) { - const uint32_t invalidRank = getInvalidRank(model.operands[operand].type); - if (invalidRank == 0) { - continue; - } - const std::string message = "mutateOperandRankTest: operand " + std::to_string(operand) + - " has rank of " + std::to_string(invalidRank); - validate(device, message, model, [operand, invalidRank](Model* model) { - model->operands[operand].dimensions = std::vector(invalidRank, 0); - }); - } -} - -///////////////////////// VALIDATE OPERAND SCALE ///////////////////////// - -static float getInvalidScale(OperandType type) { - switch (type) { - case OperandType::FLOAT16: - case OperandType::FLOAT32: - case OperandType::INT32: - case OperandType::UINT32: - case OperandType::BOOL: - case OperandType::TENSOR_BOOL8: - case OperandType::TENSOR_FLOAT16: - case OperandType::TENSOR_FLOAT32: - case OperandType::TENSOR_QUANT8_SYMM_PER_CHANNEL: - return 1.0f; - case OperandType::TENSOR_INT32: - return -1.0f; - case OperandType::TENSOR_QUANT8_SYMM: - case OperandType::TENSOR_QUANT8_ASYMM: - case OperandType::TENSOR_QUANT16_ASYMM: - case OperandType::TENSOR_QUANT16_SYMM: - return 0.0f; - default: - return 0.0f; - } -} - -static void mutateOperandScaleTest(const sp& device, const Model& model) { - for (size_t operand = 0; operand < model.operands.size(); ++operand) { - const float invalidScale = getInvalidScale(model.operands[operand].type); - const std::string message = "mutateOperandScaleTest: operand " + std::to_string(operand) + - " has scale of " + std::to_string(invalidScale); - validate(device, message, model, [operand, invalidScale](Model* model) { - model->operands[operand].scale = invalidScale; - }); - } -} - -///////////////////////// VALIDATE OPERAND ZERO POINT ///////////////////////// - -static std::vector getInvalidZeroPoints(OperandType type) { - switch (type) { - case OperandType::FLOAT16: - case OperandType::FLOAT32: - case OperandType::INT32: - case OperandType::UINT32: - case OperandType::BOOL: - case OperandType::TENSOR_BOOL8: - case OperandType::TENSOR_FLOAT16: - case OperandType::TENSOR_FLOAT32: - case OperandType::TENSOR_INT32: - case OperandType::TENSOR_QUANT8_SYMM_PER_CHANNEL: - return {1}; - case OperandType::TENSOR_QUANT8_ASYMM: - return {-1, 256}; - case OperandType::TENSOR_QUANT8_SYMM: - return {-129, -1, 1, 128}; - case OperandType::TENSOR_QUANT16_ASYMM: - return {-1, 65536}; - case OperandType::TENSOR_QUANT16_SYMM: - return {-32769, -1, 1, 32768}; - default: - return {}; - } -} - -static void mutateOperandZeroPointTest(const sp& device, const Model& model) { - for (size_t operand = 0; operand < model.operands.size(); ++operand) { - const std::vector invalidZeroPoints = - getInvalidZeroPoints(model.operands[operand].type); - for (int32_t invalidZeroPoint : invalidZeroPoints) { - const std::string message = "mutateOperandZeroPointTest: operand " + - std::to_string(operand) + " has zero point of " + - std::to_string(invalidZeroPoint); - validate(device, message, model, [operand, invalidZeroPoint](Model* model) { - model->operands[operand].zeroPoint = invalidZeroPoint; - }); - } - } -} - -///////////////////////// VALIDATE EXTRA ??? ///////////////////////// - -// TODO: Operand::lifetime -// TODO: Operand::location - -///////////////////////// VALIDATE OPERATION OPERAND TYPE ///////////////////////// - -static void mutateOperand(Operand* operand, OperandType type) { - Operand newOperand = *operand; - newOperand.type = type; - switch (type) { - case OperandType::FLOAT16: - case OperandType::FLOAT32: - case OperandType::INT32: - case OperandType::UINT32: - case OperandType::BOOL: - newOperand.dimensions = hidl_vec(); - newOperand.scale = 0.0f; - newOperand.zeroPoint = 0; - break; - case OperandType::TENSOR_BOOL8: - case OperandType::TENSOR_FLOAT16: - case OperandType::TENSOR_FLOAT32: - newOperand.dimensions = - operand->dimensions.size() > 0 ? operand->dimensions : hidl_vec({1}); - newOperand.scale = 0.0f; - newOperand.zeroPoint = 0; - break; - case OperandType::TENSOR_INT32: - newOperand.dimensions = - operand->dimensions.size() > 0 ? operand->dimensions : hidl_vec({1}); - newOperand.zeroPoint = 0; - break; - case OperandType::TENSOR_QUANT8_ASYMM: - case OperandType::TENSOR_QUANT8_SYMM: - case OperandType::TENSOR_QUANT16_ASYMM: - case OperandType::TENSOR_QUANT16_SYMM: - newOperand.dimensions = - operand->dimensions.size() > 0 ? operand->dimensions : hidl_vec({1}); - newOperand.scale = operand->scale != 0.0f ? operand->scale : 1.0f; - break; - case OperandType::TENSOR_QUANT8_SYMM_PER_CHANNEL: { - newOperand.dimensions = - operand->dimensions.size() > 0 ? operand->dimensions : hidl_vec({1}); - newOperand.scale = 0.0f; - newOperand.zeroPoint = 0; - - SymmPerChannelQuantParams channelQuant; - channelQuant.channelDim = 0; - channelQuant.scales = hidl_vec( - operand->dimensions.size() > 0 ? static_cast(operand->dimensions[0]) - : 0); - for (size_t i = 0; i < channelQuant.scales.size(); ++i) { - channelQuant.scales[i] = 1.0f; - } - newOperand.extraParams.channelQuant(std::move(channelQuant)); - } break; - case OperandType::OEM: - case OperandType::TENSOR_OEM_BYTE: - default: - break; - } - *operand = newOperand; -} - -static bool mutateOperationOperandTypeSkip(size_t operand, OperandType type, const Model& model) { - // Do not test OEM types - if (type == model.operands[operand].type || type == OperandType::OEM || - type == OperandType::TENSOR_OEM_BYTE) { - return true; - } - for (const Operation& operation : model.operations) { - // Skip mutateOperationOperandTypeTest for the following operations. - // - LSH_PROJECTION's second argument is allowed to have any type. - // - ARGMIN and ARGMAX's first argument can be any of - // TENSOR_(FLOAT16|FLOAT32|INT32|QUANT8_ASYMM). - // - CAST's argument can be any of TENSOR_(FLOAT16|FLOAT32|INT32|QUANT8_ASYMM). - // - RANDOM_MULTINOMIAL's argument can be either TENSOR_FLOAT16 or TENSOR_FLOAT32. - // - DEQUANTIZE input can be any of - // TENSOR_(QUANT8_ASYMM|QUANT8_SYMM|QUANT8_SYMM_PER_CHANNEL), output can - // be of either TENSOR_FLOAT16 or TENSOR_FLOAT32. - // - QUANTIZE input can be either TENSOR_FLOAT16 or TENSOR_FLOAT32 - // - CONV_2D filter type (arg 1) can be QUANT8_ASYMM or QUANT8_SYMM_PER_CHANNEL - // - DEPTHWISE_CONV_2D filter type (arg 1) can be QUANT8_ASYMM or QUANT8_SYMM_PER_CHANNEL - // - GROUPED_CONV_2D filter type (arg 1) can be QUANT8_ASYMM or QUANT8_SYMM_PER_CHANNEL - // - TRANSPOSE_CONV_2D filter type (arg 1) can be QUANT8_ASYMM or QUANT8_SYMM_PER_CHANNEL - switch (operation.type) { - case OperationType::LSH_PROJECTION: { - if (operand == operation.inputs[1]) { - return true; - } - } break; - case OperationType::CAST: - case OperationType::ARGMAX: - case OperationType::ARGMIN: { - if (type == OperandType::TENSOR_FLOAT16 || type == OperandType::TENSOR_FLOAT32 || - type == OperandType::TENSOR_INT32 || type == OperandType::TENSOR_QUANT8_ASYMM) { - return true; - } - } break; - case OperationType::QUANTIZE: - case OperationType::RANDOM_MULTINOMIAL: { - if (operand == operation.inputs[0] && - (type == OperandType::TENSOR_FLOAT16 || type == OperandType::TENSOR_FLOAT32)) { - return true; - } - } break; - case OperationType::DEQUANTIZE: { - if (operand == operation.inputs[0] && - (type == OperandType::TENSOR_QUANT8_ASYMM || - type == OperandType::TENSOR_QUANT8_SYMM || - type == OperandType::TENSOR_QUANT8_SYMM_PER_CHANNEL)) { - return true; - } - if (operand == operation.outputs[0] && - (type == OperandType::TENSOR_FLOAT16 || type == OperandType::TENSOR_FLOAT32)) { - return true; - } - } break; - case OperationType::TRANSPOSE_CONV_2D: - case OperationType::GROUPED_CONV_2D: - case OperationType::DEPTHWISE_CONV_2D: - case OperationType::CONV_2D: { - if (operand == operation.inputs[1] && - (type == OperandType::TENSOR_QUANT8_ASYMM || - type == OperandType::TENSOR_QUANT8_SYMM_PER_CHANNEL)) { - return true; - } - } break; - default: - break; - } - } - return false; -} - -static void mutateOperationOperandTypeTest(const sp& device, const Model& model) { - for (size_t operand = 0; operand < model.operands.size(); ++operand) { - for (OperandType invalidOperandType : hidl_enum_range{}) { - if (mutateOperationOperandTypeSkip(operand, invalidOperandType, model)) { - continue; - } - const std::string message = "mutateOperationOperandTypeTest: operand " + - std::to_string(operand) + " set to type " + - toString(invalidOperandType); - validate(device, message, model, [operand, invalidOperandType](Model* model) { - mutateOperand(&model->operands[operand], invalidOperandType); - }); - } - } -} - -///////////////////////// VALIDATE MODEL OPERATION TYPE ///////////////////////// - -static const uint32_t invalidOperationTypes[] = { - static_cast(OperationTypeRange::FUNDAMENTAL_MAX) + 1, - static_cast(OperationTypeRange::OEM_MIN) - 1, - static_cast(OperationTypeRange::OEM_MAX) + 1, -}; - -static void mutateOperationTypeTest(const sp& device, const Model& model) { - for (size_t operation = 0; operation < model.operations.size(); ++operation) { - for (uint32_t invalidOperationType : invalidOperationTypes) { - const std::string message = "mutateOperationTypeTest: operation " + - std::to_string(operation) + " set to value " + - std::to_string(invalidOperationType); - validate(device, message, model, [operation, invalidOperationType](Model* model) { - model->operations[operation].type = - static_cast(invalidOperationType); - }); - } - } -} - -///////////////////////// VALIDATE MODEL OPERATION INPUT OPERAND INDEX ///////////////////////// - -static void mutateOperationInputOperandIndexTest(const sp& device, const Model& model) { - for (size_t operation = 0; operation < model.operations.size(); ++operation) { - const uint32_t invalidOperand = model.operands.size(); - for (size_t input = 0; input < model.operations[operation].inputs.size(); ++input) { - const std::string message = "mutateOperationInputOperandIndexTest: operation " + - std::to_string(operation) + " input " + - std::to_string(input); - validate(device, message, model, [operation, input, invalidOperand](Model* model) { - model->operations[operation].inputs[input] = invalidOperand; - }); - } - } -} - -///////////////////////// VALIDATE MODEL OPERATION OUTPUT OPERAND INDEX ///////////////////////// - -static void mutateOperationOutputOperandIndexTest(const sp& device, const Model& model) { - for (size_t operation = 0; operation < model.operations.size(); ++operation) { - const uint32_t invalidOperand = model.operands.size(); - for (size_t output = 0; output < model.operations[operation].outputs.size(); ++output) { - const std::string message = "mutateOperationOutputOperandIndexTest: operation " + - std::to_string(operation) + " output " + - std::to_string(output); - validate(device, message, model, [operation, output, invalidOperand](Model* model) { - model->operations[operation].outputs[output] = invalidOperand; - }); - } - } -} - -///////////////////////// REMOVE OPERAND FROM EVERYTHING ///////////////////////// - -static void removeValueAndDecrementGreaterValues(hidl_vec* vec, uint32_t value) { - if (vec) { - // remove elements matching "value" - auto last = std::remove(vec->begin(), vec->end(), value); - vec->resize(std::distance(vec->begin(), last)); - - // decrement elements exceeding "value" - std::transform(vec->begin(), vec->end(), vec->begin(), - [value](uint32_t v) { return v > value ? v-- : v; }); - } -} - -static void removeOperand(Model* model, uint32_t index) { - hidl_vec_removeAt(&model->operands, index); - for (Operation& operation : model->operations) { - removeValueAndDecrementGreaterValues(&operation.inputs, index); - removeValueAndDecrementGreaterValues(&operation.outputs, index); - } - removeValueAndDecrementGreaterValues(&model->inputIndexes, index); - removeValueAndDecrementGreaterValues(&model->outputIndexes, index); -} - -static bool removeOperandSkip(size_t operand, const Model& model) { - for (const Operation& operation : model.operations) { - // Skip removeOperandTest for the following operations. - // - SPLIT's outputs are not checked during prepareModel. - if (operation.type == OperationType::SPLIT) { - for (const size_t outOprand : operation.outputs) { - if (operand == outOprand) { - return true; - } - } - } - // BIDIRECTIONAL_SEQUENCE_LSTM and BIDIRECTIONAL_SEQUENCE_RNN can have either one or two - // outputs depending on their mergeOutputs parameter. - if (operation.type == OperationType::BIDIRECTIONAL_SEQUENCE_LSTM || - operation.type == OperationType::BIDIRECTIONAL_SEQUENCE_RNN) { - for (const size_t outOprand : operation.outputs) { - if (operand == outOprand) { - return true; - } - } - } - } - return false; -} - -static void removeOperandTest(const sp& device, const Model& model) { - for (size_t operand = 0; operand < model.operands.size(); ++operand) { - if (removeOperandSkip(operand, model)) { - continue; - } - const std::string message = "removeOperandTest: operand " + std::to_string(operand); - validate(device, message, model, - [operand](Model* model) { removeOperand(model, operand); }); - } -} - -///////////////////////// REMOVE OPERATION ///////////////////////// - -static void removeOperation(Model* model, uint32_t index) { - for (uint32_t operand : model->operations[index].inputs) { - model->operands[operand].numberOfConsumers--; - } - hidl_vec_removeAt(&model->operations, index); -} - -static void removeOperationTest(const sp& device, const Model& model) { - for (size_t operation = 0; operation < model.operations.size(); ++operation) { - const std::string message = "removeOperationTest: operation " + std::to_string(operation); - validate(device, message, model, - [operation](Model* model) { removeOperation(model, operation); }); - } -} - -///////////////////////// REMOVE OPERATION INPUT ///////////////////////// - -static bool removeOperationInputSkip(const Operation& op, size_t input) { - // Skip removeOperationInputTest for the following operations. - // - CONCATENATION has at least 2 inputs, with the last element being INT32. - // - CONV_2D, DEPTHWISE_CONV_2D, MAX_POOL_2D, AVERAGE_POOL_2D, L2_POOL_2D, RESIZE_BILINEAR, - // SPACE_TO_DEPTH, SPACE_TO_DEPTH, SPACE_TO_BATCH_ND, BATCH_TO_SPACE_ND can have an optional - // layout parameter. - // - L2_NORMALIZATION, LOCAL_RESPONSE_NORMALIZATION, SOFTMAX can have an optional axis - // parameter. - switch (op.type) { - case OperationType::CONCATENATION: { - if (op.inputs.size() > 2 && input != op.inputs.size() - 1) { - return true; - } - } break; - case OperationType::DEPTHWISE_CONV_2D: { - if ((op.inputs.size() == 12 && input == 11) || (op.inputs.size() == 9 && input == 8)) { - return true; - } - } break; - case OperationType::CONV_2D: - case OperationType::AVERAGE_POOL_2D: - case OperationType::MAX_POOL_2D: - case OperationType::L2_POOL_2D: { - if ((op.inputs.size() == 11 && input == 10) || (op.inputs.size() == 8 && input == 7)) { - return true; - } - } break; - case OperationType::RESIZE_BILINEAR: { - if (op.inputs.size() == 4 && input == 3) { - return true; - } - } break; - case OperationType::SPACE_TO_DEPTH: - case OperationType::DEPTH_TO_SPACE: - case OperationType::BATCH_TO_SPACE_ND: { - if (op.inputs.size() == 3 && input == 2) { - return true; - } - } break; - case OperationType::SPACE_TO_BATCH_ND: { - if (op.inputs.size() == 4 && input == 3) { - return true; - } - } break; - case OperationType::L2_NORMALIZATION: { - if (op.inputs.size() == 2 && input == 1) { - return true; - } - } break; - case OperationType::LOCAL_RESPONSE_NORMALIZATION: { - if (op.inputs.size() == 6 && input == 5) { - return true; - } - } break; - case OperationType::SOFTMAX: { - if (op.inputs.size() == 3 && input == 2) { - return true; - } - } break; - default: - break; - } - return false; -} - -static void removeOperationInputTest(const sp& device, const Model& model) { - for (size_t operation = 0; operation < model.operations.size(); ++operation) { - for (size_t input = 0; input < model.operations[operation].inputs.size(); ++input) { - const Operation& op = model.operations[operation]; - if (removeOperationInputSkip(op, input)) { - continue; - } - const std::string message = "removeOperationInputTest: operation " + - std::to_string(operation) + ", input " + - std::to_string(input); - validate(device, message, model, [operation, input](Model* model) { - uint32_t operand = model->operations[operation].inputs[input]; - model->operands[operand].numberOfConsumers--; - hidl_vec_removeAt(&model->operations[operation].inputs, input); - }); - } - } -} - -///////////////////////// REMOVE OPERATION OUTPUT ///////////////////////// - -static void removeOperationOutputTest(const sp& device, const Model& model) { - for (size_t operation = 0; operation < model.operations.size(); ++operation) { - for (size_t output = 0; output < model.operations[operation].outputs.size(); ++output) { - const std::string message = "removeOperationOutputTest: operation " + - std::to_string(operation) + ", output " + - std::to_string(output); - validate(device, message, model, [operation, output](Model* model) { - hidl_vec_removeAt(&model->operations[operation].outputs, output); - }); - } - } -} - -///////////////////////// MODEL VALIDATION ///////////////////////// - -// TODO: remove model input -// TODO: remove model output -// TODO: add unused operation - -///////////////////////// ADD OPERATION INPUT ///////////////////////// - -static bool addOperationInputSkip(const Operation& op) { - // Skip addOperationInputTest for the following operations. - // - L2_NORMALIZATION, LOCAL_RESPONSE_NORMALIZATION, SOFTMAX can have an optional INT32 axis - // parameter. - if ((op.type == OperationType::L2_NORMALIZATION && op.inputs.size() == 1) || - (op.type == OperationType::LOCAL_RESPONSE_NORMALIZATION && op.inputs.size() == 5) || - (op.type == OperationType::SOFTMAX && op.inputs.size() == 2)) { - return true; - } - return false; -} - -static void addOperationInputTest(const sp& device, const Model& model) { - for (size_t operation = 0; operation < model.operations.size(); ++operation) { - if (addOperationInputSkip(model.operations[operation])) { - continue; - } - const std::string message = "addOperationInputTest: operation " + std::to_string(operation); - validate(device, message, model, [operation](Model* model) { - uint32_t index = addOperand(model, OperandLifeTime::MODEL_INPUT); - hidl_vec_push_back(&model->operations[operation].inputs, index); - hidl_vec_push_back(&model->inputIndexes, index); - }); - } -} - -///////////////////////// ADD OPERATION OUTPUT ///////////////////////// - -static void addOperationOutputTest(const sp& device, const Model& model) { - for (size_t operation = 0; operation < model.operations.size(); ++operation) { - const std::string message = - "addOperationOutputTest: operation " + std::to_string(operation); - validate(device, message, model, [operation](Model* model) { - uint32_t index = addOperand(model, OperandLifeTime::MODEL_OUTPUT); - hidl_vec_push_back(&model->operations[operation].outputs, index); - hidl_vec_push_back(&model->outputIndexes, index); - }); - } -} - -///////////////////////// VALIDATE EXECUTION PREFERENCE ///////////////////////// - -static const int32_t invalidExecutionPreferences[] = { - static_cast(ExecutionPreference::LOW_POWER) - 1, // lower bound - static_cast(ExecutionPreference::SUSTAINED_SPEED) + 1, // upper bound -}; - -static void mutateExecutionPreferenceTest(const sp& device, const Model& model) { - for (int32_t preference : invalidExecutionPreferences) { - const std::string message = - "mutateExecutionPreferenceTest: preference " + std::to_string(preference); - validate( - device, message, model, [](Model*) {}, - static_cast(preference)); - } -} - -////////////////////////// ENTRY POINT ////////////////////////////// - -void validateModel(const sp& device, const Model& model) { - mutateOperandTypeTest(device, model); - mutateOperandRankTest(device, model); - mutateOperandScaleTest(device, model); - mutateOperandZeroPointTest(device, model); - mutateOperationOperandTypeTest(device, model); - mutateOperationTypeTest(device, model); - mutateOperationInputOperandIndexTest(device, model); - mutateOperationOutputOperandIndexTest(device, model); - removeOperandTest(device, model); - removeOperationTest(device, model); - removeOperationInputTest(device, model); - removeOperationOutputTest(device, model); - addOperationInputTest(device, model); - addOperationOutputTest(device, model); - mutateExecutionPreferenceTest(device, model); -} - -} // namespace android::hardware::neuralnetworks::V1_2::vts::functional diff --git a/neuralnetworks/1.3/vts/functional/ValidateRequest.cpp b/neuralnetworks/1.3/vts/functional/ValidateRequest.cpp deleted file mode 100644 index f25ee62617..0000000000 --- a/neuralnetworks/1.3/vts/functional/ValidateRequest.cpp +++ /dev/null @@ -1,168 +0,0 @@ -/* - * Copyright (C) 2018 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#define LOG_TAG "neuralnetworks_hidl_hal_test" - -#include "1.0/Utils.h" -#include "1.2/Callbacks.h" -#include "ExecutionBurstController.h" -#include "GeneratedTestHarness.h" -#include "TestHarness.h" -#include "Utils.h" -#include "VtsHalNeuralnetworks.h" - -namespace android::hardware::neuralnetworks::V1_2::vts::functional { - -using implementation::ExecutionCallback; -using V1_0::ErrorStatus; -using V1_0::Request; - -///////////////////////// UTILITY FUNCTIONS ///////////////////////// - -static bool badTiming(Timing timing) { - return timing.timeOnDevice == UINT64_MAX && timing.timeInDriver == UINT64_MAX; -} - -// Primary validation function. This function will take a valid request, apply a -// mutation to it to invalidate the request, then pass it to interface calls -// that use the request. Note that the request here is passed by value, and any -// mutation to the request does not leave this function. -static void validate(const sp& preparedModel, const std::string& message, - Request request, const std::function& mutation) { - mutation(&request); - - // We'd like to test both with timing requested and without timing - // requested. Rather than running each test both ways, we'll decide whether - // to request timing by hashing the message. We do not use std::hash because - // it is not guaranteed stable across executions. - char hash = 0; - for (auto c : message) { - hash ^= c; - }; - MeasureTiming measure = (hash & 1) ? MeasureTiming::YES : MeasureTiming::NO; - - // asynchronous - { - SCOPED_TRACE(message + " [execute_1_2]"); - - sp executionCallback = new ExecutionCallback(); - Return executeLaunchStatus = - preparedModel->execute_1_2(request, measure, executionCallback); - ASSERT_TRUE(executeLaunchStatus.isOk()); - ASSERT_EQ(ErrorStatus::INVALID_ARGUMENT, static_cast(executeLaunchStatus)); - - executionCallback->wait(); - ErrorStatus executionReturnStatus = executionCallback->getStatus(); - const auto& outputShapes = executionCallback->getOutputShapes(); - Timing timing = executionCallback->getTiming(); - ASSERT_EQ(ErrorStatus::INVALID_ARGUMENT, executionReturnStatus); - ASSERT_EQ(outputShapes.size(), 0); - ASSERT_TRUE(badTiming(timing)); - } - - // synchronous - { - SCOPED_TRACE(message + " [executeSynchronously]"); - - Return executeStatus = preparedModel->executeSynchronously( - request, measure, - [](ErrorStatus error, const hidl_vec& outputShapes, - const Timing& timing) { - ASSERT_EQ(ErrorStatus::INVALID_ARGUMENT, error); - EXPECT_EQ(outputShapes.size(), 0); - EXPECT_TRUE(badTiming(timing)); - }); - ASSERT_TRUE(executeStatus.isOk()); - } - - // burst - { - SCOPED_TRACE(message + " [burst]"); - - // create burst - std::shared_ptr<::android::nn::ExecutionBurstController> burst = - android::nn::ExecutionBurstController::create(preparedModel, /*blocking=*/true); - ASSERT_NE(nullptr, burst.get()); - - // create memory keys - std::vector keys(request.pools.size()); - for (size_t i = 0; i < keys.size(); ++i) { - keys[i] = reinterpret_cast(&request.pools[i]); - } - - // execute and verify - ErrorStatus error; - std::vector outputShapes; - Timing timing; - std::tie(error, outputShapes, timing) = burst->compute(request, measure, keys); - EXPECT_EQ(ErrorStatus::INVALID_ARGUMENT, error); - EXPECT_EQ(outputShapes.size(), 0); - EXPECT_TRUE(badTiming(timing)); - - // additional burst testing - if (request.pools.size() > 0) { - // valid free - burst->freeMemory(keys.front()); - - // negative test: invalid free of unknown (blank) memory - burst->freeMemory(intptr_t{}); - - // negative test: double free of memory - burst->freeMemory(keys.front()); - } - } -} - -///////////////////////// REMOVE INPUT //////////////////////////////////// - -static void removeInputTest(const sp& preparedModel, const Request& request) { - for (size_t input = 0; input < request.inputs.size(); ++input) { - const std::string message = "removeInput: removed input " + std::to_string(input); - validate(preparedModel, message, request, - [input](Request* request) { hidl_vec_removeAt(&request->inputs, input); }); - } -} - -///////////////////////// REMOVE OUTPUT //////////////////////////////////// - -static void removeOutputTest(const sp& preparedModel, const Request& request) { - for (size_t output = 0; output < request.outputs.size(); ++output) { - const std::string message = "removeOutput: removed Output " + std::to_string(output); - validate(preparedModel, message, request, - [output](Request* request) { hidl_vec_removeAt(&request->outputs, output); }); - } -} - -///////////////////////////// ENTRY POINT ////////////////////////////////// - -void validateRequest(const sp& preparedModel, const Request& request) { - removeInputTest(preparedModel, request); - removeOutputTest(preparedModel, request); -} - -void validateRequestFailure(const sp& preparedModel, const Request& request) { - SCOPED_TRACE("Expecting request to fail [executeSynchronously]"); - Return executeStatus = preparedModel->executeSynchronously( - request, MeasureTiming::NO, - [](ErrorStatus error, const hidl_vec& outputShapes, const Timing& timing) { - ASSERT_NE(ErrorStatus::NONE, error); - EXPECT_EQ(outputShapes.size(), 0); - EXPECT_TRUE(badTiming(timing)); - }); - ASSERT_TRUE(executeStatus.isOk()); -} - -} // namespace android::hardware::neuralnetworks::V1_2::vts::functional diff --git a/neuralnetworks/1.3/vts/functional/VtsHalNeuralnetworks.cpp b/neuralnetworks/1.3/vts/functional/VtsHalNeuralnetworks.cpp deleted file mode 100644 index 4fbd0e270f..0000000000 --- a/neuralnetworks/1.3/vts/functional/VtsHalNeuralnetworks.cpp +++ /dev/null @@ -1,171 +0,0 @@ -/* - * Copyright (C) 2018 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#define LOG_TAG "neuralnetworks_hidl_hal_test" - -#include "VtsHalNeuralnetworks.h" -#include -#include -#include -#include -#include "1.0/Callbacks.h" -#include "1.0/Utils.h" -#include "GeneratedTestHarness.h" -#include "TestHarness.h" - -namespace android::hardware::neuralnetworks::V1_2::vts::functional { - -using implementation::PreparedModelCallback; -using HidlToken = hidl_array(Constant::BYTE_SIZE_OF_CACHE_TOKEN)>; -using V1_0::ErrorStatus; -using V1_0::Request; -using V1_1::ExecutionPreference; - -// internal helper function -void createPreparedModel(const sp& device, const Model& model, - sp* preparedModel) { - ASSERT_NE(nullptr, preparedModel); - *preparedModel = nullptr; - - // see if service can handle model - bool fullySupportsModel = false; - const Return supportedCall = device->getSupportedOperations_1_2( - model, [&fullySupportsModel](ErrorStatus status, const hidl_vec& supported) { - ASSERT_EQ(ErrorStatus::NONE, status); - ASSERT_NE(0ul, supported.size()); - fullySupportsModel = std::all_of(supported.begin(), supported.end(), - [](bool valid) { return valid; }); - }); - ASSERT_TRUE(supportedCall.isOk()); - - // launch prepare model - const sp preparedModelCallback = new PreparedModelCallback(); - const Return prepareLaunchStatus = device->prepareModel_1_2( - model, ExecutionPreference::FAST_SINGLE_ANSWER, hidl_vec(), - hidl_vec(), HidlToken(), preparedModelCallback); - ASSERT_TRUE(prepareLaunchStatus.isOk()); - ASSERT_EQ(ErrorStatus::NONE, static_cast(prepareLaunchStatus)); - - // retrieve prepared model - preparedModelCallback->wait(); - const ErrorStatus prepareReturnStatus = preparedModelCallback->getStatus(); - *preparedModel = getPreparedModel_1_2(preparedModelCallback); - - // The getSupportedOperations_1_2 call returns a list of operations that are - // guaranteed not to fail if prepareModel_1_2 is called, and - // 'fullySupportsModel' is true i.f.f. the entire model is guaranteed. - // If a driver has any doubt that it can prepare an operation, it must - // return false. So here, if a driver isn't sure if it can support an - // operation, but reports that it successfully prepared the model, the test - // can continue. - if (!fullySupportsModel && prepareReturnStatus != ErrorStatus::NONE) { - ASSERT_EQ(nullptr, preparedModel->get()); - LOG(INFO) << "NN VTS: Early termination of test because vendor service cannot prepare " - "model that it does not support."; - std::cout << "[ ] Early termination of test because vendor service cannot " - "prepare model that it does not support." - << std::endl; - GTEST_SKIP(); - } - ASSERT_EQ(ErrorStatus::NONE, prepareReturnStatus); - ASSERT_NE(nullptr, preparedModel->get()); -} - -void NeuralnetworksHidlTest::SetUp() { - testing::TestWithParam::SetUp(); - ASSERT_NE(kDevice, nullptr); -} - -static NamedDevice makeNamedDevice(const std::string& name) { - return {name, IDevice::getService(name)}; -} - -static std::vector getNamedDevicesImpl() { - // Retrieves the name of all service instances that implement IDevice, - // including any Lazy HAL instances. - const std::vector names = hardware::getAllHalInstanceNames(IDevice::descriptor); - - // Get a handle to each device and pair it with its name. - std::vector namedDevices; - namedDevices.reserve(names.size()); - std::transform(names.begin(), names.end(), std::back_inserter(namedDevices), makeNamedDevice); - return namedDevices; -} - -const std::vector& getNamedDevices() { - const static std::vector devices = getNamedDevicesImpl(); - return devices; -} - -std::string printNeuralnetworksHidlTest( - const testing::TestParamInfo& info) { - return gtestCompliantName(getName(info.param)); -} - -INSTANTIATE_DEVICE_TEST(NeuralnetworksHidlTest); - -// Forward declaration from ValidateModel.cpp -void validateModel(const sp& device, const Model& model); -// Forward declaration from ValidateRequest.cpp -void validateRequest(const sp& preparedModel, const V1_0::Request& request); -// Forward declaration from ValidateRequest.cpp -void validateRequestFailure(const sp& preparedModel, const V1_0::Request& request); -// Forward declaration from ValidateBurst.cpp -void validateBurst(const sp& preparedModel, const V1_0::Request& request); - -void validateEverything(const sp& device, const Model& model, const Request& request) { - validateModel(device, model); - - // Create IPreparedModel. - sp preparedModel; - createPreparedModel(device, model, &preparedModel); - if (preparedModel == nullptr) return; - - validateRequest(preparedModel, request); - validateBurst(preparedModel, request); -} - -void validateFailure(const sp& device, const Model& model, const Request& request) { - // TODO: Should this always succeed? - // What if the invalid input is part of the model (i.e., a parameter). - validateModel(device, model); - - // Create IPreparedModel. - sp preparedModel; - createPreparedModel(device, model, &preparedModel); - if (preparedModel == nullptr) return; - - validateRequestFailure(preparedModel, request); -} - -TEST_P(ValidationTest, Test) { - const Model model = createModel(kTestModel); - const Request request = createRequest(kTestModel); - if (kTestModel.expectFailure) { - validateFailure(kDevice, model, request); - } else { - validateEverything(kDevice, model, request); - } -} - -INSTANTIATE_GENERATED_TEST(ValidationTest, [](const test_helper::TestModel&) { return true; }); - -sp getPreparedModel_1_2(const sp& callback) { - sp preparedModelV1_0 = callback->getPreparedModel(); - return IPreparedModel::castFrom(preparedModelV1_0).withDefault(nullptr); -} - -} // namespace android::hardware::neuralnetworks::V1_2::vts::functional diff --git a/neuralnetworks/1.3/vts/functional/VtsHalNeuralnetworks.h b/neuralnetworks/1.3/vts/functional/VtsHalNeuralnetworks.h deleted file mode 100644 index d01336eccd..0000000000 --- a/neuralnetworks/1.3/vts/functional/VtsHalNeuralnetworks.h +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Copyright (C) 2018 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef ANDROID_HARDWARE_NEURALNETWORKS_V1_2_VTS_HAL_NEURALNETWORKS_H -#define ANDROID_HARDWARE_NEURALNETWORKS_V1_2_VTS_HAL_NEURALNETWORKS_H - -#include -#include -#include -#include -#include "1.0/Utils.h" -#include "1.2/Callbacks.h" - -namespace android::hardware::neuralnetworks::V1_2::vts::functional { - -using NamedDevice = Named>; -using NeuralnetworksHidlTestParam = NamedDevice; - -class NeuralnetworksHidlTest : public testing::TestWithParam { - protected: - void SetUp() override; - const sp kDevice = getData(GetParam()); -}; - -const std::vector& getNamedDevices(); - -std::string printNeuralnetworksHidlTest( - const testing::TestParamInfo& info); - -#define INSTANTIATE_DEVICE_TEST(TestSuite) \ - INSTANTIATE_TEST_SUITE_P(PerInstance, TestSuite, testing::ValuesIn(getNamedDevices()), \ - printNeuralnetworksHidlTest) - -// Create an IPreparedModel object. If the model cannot be prepared, -// "preparedModel" will be nullptr instead. -void createPreparedModel(const sp& device, const Model& model, - sp* preparedModel); - -// Utility function to get PreparedModel from callback and downcast to V1_2. -sp getPreparedModel_1_2(const sp& callback); - -} // namespace android::hardware::neuralnetworks::V1_2::vts::functional - -#endif // ANDROID_HARDWARE_NEURALNETWORKS_V1_2_VTS_HAL_NEURALNETWORKS_H diff --git a/neuralnetworks/1.3/vts/functional/include/1.2/Callbacks.h b/neuralnetworks/1.3/vts/functional/include/1.2/Callbacks.h deleted file mode 100644 index bf4792cc6b..0000000000 --- a/neuralnetworks/1.3/vts/functional/include/1.2/Callbacks.h +++ /dev/null @@ -1,325 +0,0 @@ -/* - * Copyright (C) 2018 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef ANDROID_HARDWARE_NEURALNETWORKS_V1_2_CALLBACKS_H -#define ANDROID_HARDWARE_NEURALNETWORKS_V1_2_CALLBACKS_H - -#include -#include -#include -#include -#include -#include -#include -#include - -/* - * The Callback classes are used internally by the NeuralNetworks runtime to - * synchronize between different threads. An asynchronous task is launched - * paired with a callback object. When a client thread requires the output being - * generated by the asynchronous task, the client thread can wait for the result - * and be blocked until it has completed. Any wait may safely be called - * concurrently, even on the same callback object. When the asynchronous task - * has finished its workload, it must immediately call "notify*". If the - * asynchronous task has failed to launch, the function that tried to launch the - * asynchronous task must immediately call "notify*". This "notify*" call - * awakens any client threads waiting on the callback object. - * - * These classes exist to enable synchronization across HIDL. When - * synchronization is only required in the same process, consider using - * std::future, std::mutex, std::condition_variable, or std::experimental::latch - * instead. - */ - -namespace android::hardware::neuralnetworks::V1_2::implementation { - -/** - * The PreparedModelCallback class is used to receive the error status of - * preparing a model as well as the prepared model from a task executing - * asynchronously with respect to the runtime. If a calling thread calls wait - * or get* on a PreparedModelCallback object and the corresponding asynchronous - * task has not finished preparing the model, the calling thread will block - * until the asynchronous task has either called notify or notify_1_2. - * - * If the callback object is notified more than once, only the results of the - * first call to notify* are used, and the results from subsequent calls are - * discarded. - * - * This callback object is passed as an argument to IDevice::prepareModel*. - */ -class PreparedModelCallback : public IPreparedModelCallback { - public: - /** - * IPreparedModelCallback::notify marks the callback object with the return - * status of the asynchronous model preparation along with the prepared - * model, and allows all prior and future wait calls on the - * PreparedModelCallback object to proceed. - * - * Either IPreparedModelCallback::notify or - * IPreparedModelCallback::notify_1_2 must be called on a given - * PreparedModelCallback object. - * - * If the callback object is notified more than once, only the results of - * the first call to notify* are used, and the results from subsequent calls - * are discarded. - * - * @param status Error status returned from asynchronously preparing the - * model; will be: - * - NONE if the asynchronous preparation was successful - * - DEVICE_UNAVAILABLE if driver is offline or busy - * - GENERAL_FAILURE if there is an unspecified error - * - INVALID_ARGUMENT if the input model is invalid - * @param preparedModel Returned model that has been prepared for execution, - * nullptr if the model was unable to be prepared. - */ - Return notify(V1_0::ErrorStatus status, - const sp& preparedModel) override; - - /** - * IPreparedModelCallback::notify_1_2 marks the callback object with the - * return status of the asynchronous model preparation along with the - * prepared model, and allows all prior and future wait calls on the - * PreparedModelCallback object to proceed. - * - * Either IPreparedModelCallback::notify or - * IPreparedModelCallback::notify_1_2 must be called on a given - * PreparedModelCallback object. - * - * If the callback object is notified more than once, only the results of - * the first call to notify* are used, and the results from subsequent calls - * are discarded. - * - * @param status Error status returned from asynchronously preparing the - * model; will be: - * - NONE if the asynchronous preparation was successful - * - DEVICE_UNAVAILABLE if driver is offline or busy - * - GENERAL_FAILURE if there is an unspecified error - * - INVALID_ARGUMENT if the input model is invalid - * @param preparedModel Returned model that has been prepared for execution, - * nullptr if the model was unable to be prepared. - */ - Return notify_1_2(V1_0::ErrorStatus status, - const sp& preparedModel) override; - - /** - * PreparedModelCallback::wait blocks until notify* has been called on the - * callback object. - */ - void wait() const; - - /** - * Retrieves the error status returned from the asynchronous task launched - * by IDevice::prepareModel*. If IDevice::prepareModel* has not finished - * asynchronously preparing the model, this call will block until the - * asynchronous task notifies the object. - * - * @return status Error status returned from asynchronously preparing the - * model; will be: - * - NONE if the asynchronous preparation was successful - * - DEVICE_UNAVAILABLE if driver is offline or busy - * - GENERAL_FAILURE if there is an unspecified error - * - INVALID_ARGUMENT if the input model is invalid - */ - V1_0::ErrorStatus getStatus() const; - - /** - * Retrieves the model that has been prepared for execution from the - * asynchronous task launched by IDevice::prepareModel*. If - * IDevice::prepareModel* has not finished asynchronously preparing the - * model, this call will block until the asynchronous task notifies the - * object. - * - * @return preparedModel Returned model that has been prepared for - * execution, nullptr if the model was unable to be prepared. - */ - sp getPreparedModel() const; - - private: - mutable std::mutex mMutex; - mutable std::condition_variable mCondition; - bool mNotified GUARDED_BY(mMutex) = false; - V1_0::ErrorStatus mErrorStatus = V1_0::ErrorStatus::GENERAL_FAILURE; - sp mPreparedModel; -}; - -/** - * The ExecutionCallback class is used to receive the results of the execution - * from a task executing asynchronously with respect to the runtime. If a - * calling thread calls wait or get* on a ExecutionCallback object and the - * corresponding asynchronous task has not finished the execution, the calling - * thread will block until the asynchronous task has either called notify or - * notify_1_2. - * - * If the callback object is notified more than once, only the results of the - * first call to notify* are used, and the results from subsequent calls are - * discarded. - * - * This callback object is passed as an argument to IPreparedModel::execute*. - */ -class ExecutionCallback : public IExecutionCallback { - public: - /** - * IExecutionCallback::notify marks the callback object with the return - * status of the asynchronous execution that held this callback and enables - * all prior and future wait calls on the ExecutionCallback object to - * proceed. - * - * Either IExecutionCallback::notify or IExecutionCallback::notify_1_2 must - * be called on a given ExecutionCallback object. - * - * If the callback object is notified more than once, only the results of - * the first call to notify* are used, and the results from subsequent calls - * are discarded. - * - * @param status Error status returned from launching the asynchronous task - * (if the launch fails) or from the asynchronous task itself (if the - * launch succeeds). Must be: - * - NONE if the asynchronous execution was successful - * - DEVICE_UNAVAILABLE if driver is offline or busy - * - GENERAL_FAILURE if there is an unspecified error - * - OUTPUT_INSUFFICIENT_SIZE if provided output buffer is not large - * enough to store the resultant values - * - INVALID_ARGUMENT if the input request is invalid - */ - Return notify(V1_0::ErrorStatus status) override; - - /** - * IExecutionCallback::notify_1_2 marks the callback object with the results - * (error status, dynamic output shapes, and timing information) of the - * asynchronous execution that held this callback and enables all prior and - * future wait calls on the ExecutionCallback object to proceed. - * - * Either IExecutionCallback::notify or IExecutionCallback::notify_1_2 must - * be called on a given ExecutionCallback object. - * - * If the callback object is notified more than once, only the results of - * the first call to notify* are used, and the results from subsequent calls - * are discarded. - * - * @param status Error status returned from launching the asynchronous task - * (if the launch fails) or from the asynchronous task itself (if the - * launch succeeds). Must be: - * - NONE if the asynchronous execution was successful - * - DEVICE_UNAVAILABLE if driver is offline or busy - * - GENERAL_FAILURE if the asynchronous task resulted in an unspecified - * error - * - OUTPUT_INSUFFICIENT_SIZE if at least one output operand buffer is - * not large enough to store the corresponding output - * - INVALID_ARGUMENT if one of the input arguments to prepareModel is - * invalid - * @param outputShapes A list of shape information of model output operands. - * The index into "outputShapes" corresponds to the index of the output - * operand in the Request outputs vector. outputShapes must be empty - * unless the status is either NONE or OUTPUT_INSUFFICIENT_SIZE. - * @param Timing Duration of execution. Unless MeasureTiming::YES was passed - * when launching the execution and status is NONE, all times must be - * reported as UINT64_MAX. A driver may choose to report any time as - * UINT64_MAX, indicating that particular measurement is not available. - */ - Return notify_1_2(V1_0::ErrorStatus status, const hidl_vec& outputShapes, - const Timing& timing) override; - - // An overload of the latest notify interface to hide the version from ExecutionBuilder. - Return notify(V1_0::ErrorStatus status, const hidl_vec& outputShapes, - const Timing& timing) { - return notify_1_2(status, outputShapes, timing); - } - - /** - * ExecutionCallback::wait blocks until notify* has been called on the - * callback object. - */ - void wait() const; - - /** - * Retrieves the error status returned from the asynchronous task launched - * by either IPreparedModel::execute or IPreparedModel::execute_1_2. If - * IPreparedModel::execute or IPreparedModel::execute_1_2 has not finished - * asynchronously executing, this call will block until the asynchronous - * task notifies the object. - * - * @return status Error status returned from launching the asynchronous task - * (if the launch fails) or from the asynchronous task itself (if the - * launch succeeds). Must be: - * - NONE if the asynchronous execution was successful - * - DEVICE_UNAVAILABLE if driver is offline or busy - * - GENERAL_FAILURE if the asynchronous task resulted in an unspecified - * error - * - OUTPUT_INSUFFICIENT_SIZE if at least one output operand buffer is - * not large enough to store the corresponding output - * - INVALID_ARGUMENT if one of the input arguments to prepareModel is - * invalid - */ - V1_0::ErrorStatus getStatus() const; - - /** - * Retrieves the output shapes returned from the asynchronous task launched - * by IPreparedModel::execute_1_2. If IPreparedModel::execute_1_2 has not - * finished asynchronously executing, this call will block until the - * asynchronous task notifies the object. - * - * If the asynchronous task was launched by IPreparedModel::execute, an - * empty vector will be returned. - * - * @return outputShapes A list of shape information of model output - * operands. The index into "outputShapes" corresponds to the index of - * the output operand in the Request outputs vector. outputShapes must - * be empty unless the status is either NONE or - * OUTPUT_INSUFFICIENT_SIZE. outputShaps may be empty if the status is - * NONE and all model output operands are fully-specified at execution - * time. outputShapes must have the same number of elements as the - * number of model output operands if the status is - * OUTPUT_INSUFFICIENT_SIZE, or if the status is NONE and the model has - * at least one output operand that is not fully-specified. - */ - const std::vector& getOutputShapes() const; - - /** - * Retrieves the duration of execution of the asynchronous task launched by - * IPreparedModel::execute_1_2. If IPreparedModel::execute_1_2 has not - * finished asynchronously executing, this call will block until the - * asynchronous task notifies the object. - * - * If the asynchronous task was launched by IPreparedModel::execute, every - * time must be UINT64_MAX. - * - * @return timing Duration of the execution. Every time must be UINT64_MAX - * unless the status is NONE. - */ - Timing getTiming() const; - - private: - /* - * ExecutionCallback::notifyInternal stores the results of the execution - * (status, output shapes, and timing information) in the ExecutionCallback - * object before any call to wait or get* return. It then enables all prior - * and future wait calls on the ExecutionCallback object to proceed. - */ - void notifyInternal(V1_0::ErrorStatus errorStatus, const hidl_vec& outputShapes, - const Timing& timing); - - // members - mutable std::mutex mMutex; - mutable std::condition_variable mCondition; - bool mNotified GUARDED_BY(mMutex) = false; - V1_0::ErrorStatus mErrorStatus = V1_0::ErrorStatus::GENERAL_FAILURE; - std::vector mOutputShapes = {}; - Timing mTiming = {}; -}; - -} // namespace android::hardware::neuralnetworks::V1_2::implementation - -#endif // ANDROID_HARDWARE_NEURALNETWORKS_V1_2_CALLBACKS_H