From 362dfd64d5c2a431dc5138f0ac6dab4f0c23f690 Mon Sep 17 00:00:00 2001 From: Xusong Wang Date: Tue, 9 Jun 2020 10:42:12 -0700 Subject: [PATCH] Allow compilations in TOCTOU tests to fail with GENERAL_FAILURE. A compilation failure is not related to the security aspect of the TOCTOU test, but it will skip one iteration of security testing. This CL allows the compilation to fail with GENERAL_FAILURE in TOCTOU tests, and issues a retry once it happens to ensure enough test coverage. Bug: 157489048 Test: 1.2/1.3 VTS Change-Id: Idc88e0365c5d2799187093b6fd7b4abf8f8b463d --- .../functional/CompilationCachingTests.cpp | 47 +++++++++++++++---- .../functional/CompilationCachingTests.cpp | 47 +++++++++++++++---- 2 files changed, 76 insertions(+), 18 deletions(-) diff --git a/neuralnetworks/1.2/vts/functional/CompilationCachingTests.cpp b/neuralnetworks/1.2/vts/functional/CompilationCachingTests.cpp index 449b8f369d..16b313a855 100644 --- a/neuralnetworks/1.2/vts/functional/CompilationCachingTests.cpp +++ b/neuralnetworks/1.2/vts/functional/CompilationCachingTests.cpp @@ -315,7 +315,8 @@ class CompilationCachingTestBase : public testing::Test { void saveModelToCache(const Model& model, const hidl_vec& modelCache, const hidl_vec& dataCache, - sp* preparedModel = nullptr) { + sp* preparedModel = nullptr, + bool allowGeneralFailure = false) { if (preparedModel != nullptr) *preparedModel = nullptr; // Launch prepare model. @@ -329,7 +330,10 @@ class CompilationCachingTestBase : public testing::Test { // Retrieve prepared model. preparedModelCallback->wait(); - ASSERT_EQ(preparedModelCallback->getStatus(), ErrorStatus::NONE); + const auto prepareCallbackStatus = preparedModelCallback->getStatus(); + if (!allowGeneralFailure || prepareCallbackStatus != ErrorStatus::GENERAL_FAILURE) { + ASSERT_EQ(prepareCallbackStatus, ErrorStatus::NONE); + } if (preparedModel != nullptr) { *preparedModel = IPreparedModel::castFrom(preparedModelCallback->getPreparedModel()) .withDefault(nullptr); @@ -1022,7 +1026,8 @@ static void copyCacheFiles(const std::vector>& from, // Number of operations in the large test model. constexpr uint32_t kLargeModelSize = 100; -constexpr uint32_t kNumIterationsTOCTOU = 100; +constexpr uint32_t kNumSuccessfulIterationsTOCTOU = 100; +constexpr uint32_t kMaxNumFailedIterationsTOCTOU = 100; TEST_P(CompilationCachingTest, SaveToCache_TOCTOU) { if (!mIsCachingSupported) return; @@ -1050,18 +1055,30 @@ TEST_P(CompilationCachingTest, SaveToCache_TOCTOU) { // Use a different token for modelAdd. mToken[0]++; - // This test is probabilistic, so we run it multiple times. - for (uint32_t i = 0; i < kNumIterationsTOCTOU; i++) { + // This test is probabilistic, so we run it multiple times. We allow the compilation to fail + // because it is not related to the security aspect of the TOCTOU test. However, we need to have + // enough successful iterations to ensure the test coverage. + uint32_t numSuccessfulIterations = 0, numFailedIterations = 0; + while (numSuccessfulIterations < kNumSuccessfulIterationsTOCTOU) { // Save the modelAdd compilation to cache. { hidl_vec modelCache, dataCache; createCacheHandles(mModelCache, AccessMode::READ_WRITE, &modelCache); createCacheHandles(mDataCache, AccessMode::READ_WRITE, &dataCache); + sp preparedModel = nullptr; // Spawn a thread to copy the cache content concurrently while saving to cache. std::thread thread(copyCacheFiles, std::cref(modelCacheMul), std::cref(mModelCache)); - saveModelToCache(modelAdd, modelCache, dataCache); + saveModelToCache(modelAdd, modelCache, dataCache, &preparedModel, + /*allowGeneralFailure=*/true); thread.join(); + + if (preparedModel == nullptr) { + numFailedIterations++; + ASSERT_LE(numFailedIterations, kMaxNumFailedIterationsTOCTOU); + } else { + numSuccessfulIterations++; + } } // Retrieve preparedModel from cache. @@ -1112,14 +1129,26 @@ TEST_P(CompilationCachingTest, PrepareFromCache_TOCTOU) { // Use a different token for modelAdd. mToken[0]++; - // This test is probabilistic, so we run it multiple times. - for (uint32_t i = 0; i < kNumIterationsTOCTOU; i++) { + // This test is probabilistic, so we run it multiple times. We allow the compilation to fail + // because it is not related to the security aspect of the TOCTOU test. However, we need to have + // enough successful iterations to ensure the test coverage. + uint32_t numSuccessfulIterations = 0, numFailedIterations = 0; + while (numSuccessfulIterations < kNumSuccessfulIterationsTOCTOU) { // Save the modelAdd compilation to cache. { hidl_vec modelCache, dataCache; createCacheHandles(mModelCache, AccessMode::READ_WRITE, &modelCache); createCacheHandles(mDataCache, AccessMode::READ_WRITE, &dataCache); - saveModelToCache(modelAdd, modelCache, dataCache); + sp preparedModel = nullptr; + saveModelToCache(modelAdd, modelCache, dataCache, &preparedModel, + /*allowGeneralFailure=*/true); + + if (preparedModel == nullptr) { + numFailedIterations++; + ASSERT_LE(numFailedIterations, kMaxNumFailedIterationsTOCTOU); + } else { + numSuccessfulIterations++; + } } // Retrieve preparedModel from cache. diff --git a/neuralnetworks/1.3/vts/functional/CompilationCachingTests.cpp b/neuralnetworks/1.3/vts/functional/CompilationCachingTests.cpp index ac18c8ffcc..382fc767f7 100644 --- a/neuralnetworks/1.3/vts/functional/CompilationCachingTests.cpp +++ b/neuralnetworks/1.3/vts/functional/CompilationCachingTests.cpp @@ -318,7 +318,8 @@ class CompilationCachingTestBase : public testing::Test { void saveModelToCache(const Model& model, const hidl_vec& modelCache, const hidl_vec& dataCache, - sp* preparedModel = nullptr) { + sp* preparedModel = nullptr, + bool allowGeneralFailure = false) { if (preparedModel != nullptr) *preparedModel = nullptr; // Launch prepare model. @@ -332,7 +333,10 @@ class CompilationCachingTestBase : public testing::Test { // Retrieve prepared model. preparedModelCallback->wait(); - ASSERT_EQ(preparedModelCallback->getStatus(), ErrorStatus::NONE); + const auto prepareCallbackStatus = preparedModelCallback->getStatus(); + if (!allowGeneralFailure || prepareCallbackStatus != ErrorStatus::GENERAL_FAILURE) { + ASSERT_EQ(prepareCallbackStatus, ErrorStatus::NONE); + } if (preparedModel != nullptr) { *preparedModel = IPreparedModel::castFrom(preparedModelCallback->getPreparedModel()) .withDefault(nullptr); @@ -1013,7 +1017,8 @@ static void copyCacheFiles(const std::vector>& from, // Number of operations in the large test model. constexpr uint32_t kLargeModelSize = 100; -constexpr uint32_t kNumIterationsTOCTOU = 100; +constexpr uint32_t kNumSuccessfulIterationsTOCTOU = 100; +constexpr uint32_t kMaxNumFailedIterationsTOCTOU = 100; TEST_P(CompilationCachingTest, SaveToCache_TOCTOU) { if (!mIsCachingSupported) return; @@ -1041,18 +1046,30 @@ TEST_P(CompilationCachingTest, SaveToCache_TOCTOU) { // Use a different token for modelAdd. mToken[0]++; - // This test is probabilistic, so we run it multiple times. - for (uint32_t i = 0; i < kNumIterationsTOCTOU; i++) { + // This test is probabilistic, so we run it multiple times. We allow the compilation to fail + // because it is not related to the security aspect of the TOCTOU test. However, we need to have + // enough successful iterations to ensure the test coverage. + uint32_t numSuccessfulIterations = 0, numFailedIterations = 0; + while (numSuccessfulIterations < kNumSuccessfulIterationsTOCTOU) { // Save the modelAdd compilation to cache. { hidl_vec modelCache, dataCache; createCacheHandles(mModelCache, AccessMode::READ_WRITE, &modelCache); createCacheHandles(mDataCache, AccessMode::READ_WRITE, &dataCache); + sp preparedModel = nullptr; // Spawn a thread to copy the cache content concurrently while saving to cache. std::thread thread(copyCacheFiles, std::cref(modelCacheMul), std::cref(mModelCache)); - saveModelToCache(modelAdd, modelCache, dataCache); + saveModelToCache(modelAdd, modelCache, dataCache, &preparedModel, + /*allowGeneralFailure=*/true); thread.join(); + + if (preparedModel == nullptr) { + numFailedIterations++; + ASSERT_LE(numFailedIterations, kMaxNumFailedIterationsTOCTOU); + } else { + numSuccessfulIterations++; + } } // Retrieve preparedModel from cache. @@ -1103,14 +1120,26 @@ TEST_P(CompilationCachingTest, PrepareFromCache_TOCTOU) { // Use a different token for modelAdd. mToken[0]++; - // This test is probabilistic, so we run it multiple times. - for (uint32_t i = 0; i < kNumIterationsTOCTOU; i++) { + // This test is probabilistic, so we run it multiple times. We allow the compilation to fail + // because it is not related to the security aspect of the TOCTOU test. However, we need to have + // enough successful iterations to ensure the test coverage. + uint32_t numSuccessfulIterations = 0, numFailedIterations = 0; + while (numSuccessfulIterations < kNumSuccessfulIterationsTOCTOU) { // Save the modelAdd compilation to cache. { hidl_vec modelCache, dataCache; createCacheHandles(mModelCache, AccessMode::READ_WRITE, &modelCache); createCacheHandles(mDataCache, AccessMode::READ_WRITE, &dataCache); - saveModelToCache(modelAdd, modelCache, dataCache); + sp preparedModel = nullptr; + saveModelToCache(modelAdd, modelCache, dataCache, &preparedModel, + /*allowGeneralFailure=*/true); + + if (preparedModel == nullptr) { + numFailedIterations++; + ASSERT_LE(numFailedIterations, kMaxNumFailedIterationsTOCTOU); + } else { + numSuccessfulIterations++; + } } // Retrieve preparedModel from cache.