diff --git a/neuralnetworks/1.3/vts/functional/GeneratedTestHarness.cpp b/neuralnetworks/1.3/vts/functional/GeneratedTestHarness.cpp index 89edfb713c..b04abe219b 100644 --- a/neuralnetworks/1.3/vts/functional/GeneratedTestHarness.cpp +++ b/neuralnetworks/1.3/vts/functional/GeneratedTestHarness.cpp @@ -74,7 +74,7 @@ namespace { enum class Executor { ASYNC, SYNC, BURST, FENCED }; -enum class OutputType { FULLY_SPECIFIED, UNSPECIFIED, INSUFFICIENT }; +enum class OutputType { FULLY_SPECIFIED, UNSPECIFIED, INSUFFICIENT, MISSED_DEADLINE }; enum class MemoryType { SHARED, DEVICE }; @@ -495,16 +495,18 @@ static std::vector getOutputBuffers(const TestModel& testModel, cons static Return ExecutePreparedModel(const sp& preparedModel, const Request& request, MeasureTiming measure, + const OptionalTimeoutDuration& loopTimeoutDuration, sp& callback) { - return preparedModel->execute_1_3(request, measure, {}, {}, callback); + return preparedModel->execute_1_3(request, measure, {}, loopTimeoutDuration, callback); } static Return ExecutePreparedModel(const sp& preparedModel, const Request& request, MeasureTiming measure, + const OptionalTimeoutDuration& loopTimeoutDuration, hidl_vec* outputShapes, Timing* timing) { ErrorStatus result; Return ret = preparedModel->executeSynchronously_1_3( - request, measure, {}, {}, + request, measure, {}, loopTimeoutDuration, [&result, outputShapes, timing](ErrorStatus error, const hidl_vec& shapes, const Timing& time) { result = error; @@ -545,6 +547,17 @@ void EvaluatePreparedModel(const sp& device, const sp& makeOutputInsufficientSize(/*outputIndex=*/0, &request); } + OptionalTimeoutDuration loopTimeoutDuration; + // OutputType::MISSED_DEADLINE is only used by + // TestKind::INTINITE_LOOP_TIMEOUT tests to verify that an infinite loop is + // aborted after a timeout. + if (testConfig.outputType == OutputType::MISSED_DEADLINE) { + // Override the default loop timeout duration with a small value to + // speed up test execution. + constexpr uint64_t kMillisecond = 1'000'000; + loopTimeoutDuration.nanoseconds(1 * kMillisecond); + } + ErrorStatus executionStatus; hidl_vec outputShapes; Timing timing; @@ -554,8 +567,9 @@ void EvaluatePreparedModel(const sp& device, const sp& // launch execution sp executionCallback = new ExecutionCallback(); - Return executionLaunchStatus = ExecutePreparedModel( - preparedModel, request, testConfig.measureTiming, executionCallback); + Return executionLaunchStatus = + ExecutePreparedModel(preparedModel, request, testConfig.measureTiming, + loopTimeoutDuration, executionCallback); ASSERT_TRUE(executionLaunchStatus.isOk()); EXPECT_EQ(ErrorStatus::NONE, static_cast(executionLaunchStatus)); @@ -571,8 +585,9 @@ void EvaluatePreparedModel(const sp& device, const sp& SCOPED_TRACE("synchronous"); // execute - Return executionReturnStatus = ExecutePreparedModel( - preparedModel, request, testConfig.measureTiming, &outputShapes, &timing); + Return executionReturnStatus = + ExecutePreparedModel(preparedModel, request, testConfig.measureTiming, + loopTimeoutDuration, &outputShapes, &timing); ASSERT_TRUE(executionReturnStatus.isOk()); executionStatus = static_cast(executionReturnStatus); @@ -612,7 +627,7 @@ void EvaluatePreparedModel(const sp& device, const sp& hidl_handle syncFenceHandle; sp fencedCallback; Return ret = preparedModel->executeFenced( - request, {}, testConfig.measureTiming, {}, {}, {}, + request, {}, testConfig.measureTiming, {}, loopTimeoutDuration, {}, [&result, &syncFenceHandle, &fencedCallback]( ErrorStatus error, const hidl_handle& handle, const sp& callback) { @@ -686,6 +701,11 @@ void EvaluatePreparedModel(const sp& device, const sp& ASSERT_EQ(outputShapes.size(), testModel.main.outputIndexes.size()); ASSERT_FALSE(outputShapes[0].isSufficient); return; + case OutputType::MISSED_DEADLINE: + ASSERT_TRUE(executionStatus == ErrorStatus::MISSED_DEADLINE_TRANSIENT || + executionStatus == ErrorStatus::MISSED_DEADLINE_PERSISTENT) + << "executionStatus = " << executionStatus; + return; } // Go through all outputs, check returned output shapes. @@ -736,6 +756,12 @@ void EvaluatePreparedModel(const sp& device, const sp& LOG(FATAL) << "Wrong TestKind for EvaluatePreparedModel"; return; } break; + case TestKind::INTINITE_LOOP_TIMEOUT: { + outputTypesList = {OutputType::MISSED_DEADLINE}; + measureTimingList = {MeasureTiming::NO, MeasureTiming::YES}; + // Burst does not support V1_3 loop timeout. + executorList = {Executor::ASYNC, Executor::SYNC, Executor::FENCED}; + } break; } for (const OutputType outputType : outputTypesList) { @@ -794,7 +820,8 @@ void Execute(const sp& device, const TestModel& testModel, TestKind tes case TestKind::GENERAL: case TestKind::DYNAMIC_SHAPE: case TestKind::MEMORY_DOMAIN: - case TestKind::FENCED_COMPUTE: { + case TestKind::FENCED_COMPUTE: + case TestKind::INTINITE_LOOP_TIMEOUT: { createPreparedModel(device, model, &preparedModel); if (preparedModel == nullptr) return; EvaluatePreparedModel(device, preparedModel, testModel, testKind); @@ -863,24 +890,31 @@ class FencedComputeTest : public GeneratedTest {}; // Tag for the dynamic output shape tests class QuantizationCouplingTest : public GeneratedTest {}; +// Tag for the loop timeout tests +class InfiniteLoopTimeoutTest : public GeneratedTest {}; + TEST_P(GeneratedTest, Test) { - Execute(kDevice, kTestModel, /*testKind=*/TestKind::GENERAL); + Execute(kDevice, kTestModel, TestKind::GENERAL); } TEST_P(DynamicOutputShapeTest, Test) { - Execute(kDevice, kTestModel, /*testKind=*/TestKind::DYNAMIC_SHAPE); + Execute(kDevice, kTestModel, TestKind::DYNAMIC_SHAPE); } TEST_P(MemoryDomainTest, Test) { - Execute(kDevice, kTestModel, /*testKind=*/TestKind::MEMORY_DOMAIN); + Execute(kDevice, kTestModel, TestKind::MEMORY_DOMAIN); } TEST_P(FencedComputeTest, Test) { - Execute(kDevice, kTestModel, /*testKind=*/TestKind::FENCED_COMPUTE); + Execute(kDevice, kTestModel, TestKind::FENCED_COMPUTE); } TEST_P(QuantizationCouplingTest, Test) { - Execute(kDevice, kTestModel, /*testKind=*/TestKind::QUANTIZATION_COUPLING); + Execute(kDevice, kTestModel, TestKind::QUANTIZATION_COUPLING); +} + +TEST_P(InfiniteLoopTimeoutTest, Test) { + Execute(kDevice, kTestModel, TestKind::INTINITE_LOOP_TIMEOUT); } INSTANTIATE_GENERATED_TEST(GeneratedTest, @@ -900,4 +934,8 @@ INSTANTIATE_GENERATED_TEST(QuantizationCouplingTest, [](const TestModel& testMod return testModel.hasQuant8CoupledOperands() && testModel.main.operations.size() == 1; }); +INSTANTIATE_GENERATED_TEST(InfiniteLoopTimeoutTest, [](const TestModel& testModel) { + return testModel.isInfiniteLoopTimeoutTest(); +}); + } // namespace android::hardware::neuralnetworks::V1_3::vts::functional diff --git a/neuralnetworks/1.3/vts/functional/GeneratedTestHarness.h b/neuralnetworks/1.3/vts/functional/GeneratedTestHarness.h index e597fac7cf..a8db5155b5 100644 --- a/neuralnetworks/1.3/vts/functional/GeneratedTestHarness.h +++ b/neuralnetworks/1.3/vts/functional/GeneratedTestHarness.h @@ -70,7 +70,9 @@ enum class TestKind { // Tests if quantized model with TENSOR_QUANT8_ASYMM produces the same result // (OK/SKIPPED/FAILED) as the model with all such tensors converted to // TENSOR_QUANT8_ASYMM_SIGNED. - QUANTIZATION_COUPLING + QUANTIZATION_COUPLING, + // Runs a test model and verifies that MISSED_DEADLINE_* is returned. + INTINITE_LOOP_TIMEOUT }; void EvaluatePreparedModel(const sp& device, const sp& preparedModel,