From cd9af6c1cecd166a5efab0c97619006341ceedd3 Mon Sep 17 00:00:00 2001 From: Shuzhen Wang Date: Thu, 9 Sep 2021 17:42:20 -0700 Subject: [PATCH] Camera: Add version 3.8 of ICameraDeviceCallback The new version of ICameraDeviceCallback supports shutter notify with readout timestamp. The readout timestamp is used to re-time viewfinder frames to reduce jitter. When camera runs at fixed frame rate, the start_of_exposure time intervals may change, but start_or_readout intervals are constant. Test: Camera VTS test Bug: 189380857 Change-Id: I4c3856a4e30caa8cdf8417d437de1de3190c6fd6 --- camera/device/3.8/Android.bp | 38 ++++++++ camera/device/3.8/ICameraDevice.hal | 32 +++++++ camera/device/3.8/ICameraDeviceCallback.hal | 34 ++++++++ camera/device/3.8/types.hal | 68 +++++++++++++++ camera/provider/2.4/vts/functional/Android.bp | 1 + .../VtsHalCameraProviderV2_4TargetTest.cpp | 87 ++++++++++++++++++- 6 files changed, 257 insertions(+), 3 deletions(-) create mode 100644 camera/device/3.8/Android.bp create mode 100644 camera/device/3.8/ICameraDevice.hal create mode 100644 camera/device/3.8/ICameraDeviceCallback.hal create mode 100644 camera/device/3.8/types.hal diff --git a/camera/device/3.8/Android.bp b/camera/device/3.8/Android.bp new file mode 100644 index 0000000000..2a1f215023 --- /dev/null +++ b/camera/device/3.8/Android.bp @@ -0,0 +1,38 @@ +// This file is autogenerated by hidl-gen -Landroidbp. + +package { + // See: http://go/android-license-faq + // A large-scale-change added 'default_applicable_licenses' to import + // all of the 'license_kinds' from "hardware_interfaces_license" + // to get the below license kinds: + // SPDX-license-identifier-Apache-2.0 + default_applicable_licenses: ["hardware_interfaces_license"], +} + + +hidl_interface { + name: "android.hardware.camera.device@3.8", + root: "android.hardware", + srcs: [ + "types.hal", + "ICameraDevice.hal", + "ICameraDeviceCallback.hal", + ], + interfaces: [ + "android.hardware.camera.common@1.0", + "android.hardware.camera.device@3.2", + "android.hardware.camera.device@3.3", + "android.hardware.camera.device@3.4", + "android.hardware.camera.device@3.5", + "android.hardware.camera.device@3.6", + "android.hardware.camera.device@3.7", + "android.hardware.camera.metadata@3.2", + "android.hardware.camera.metadata@3.3", + "android.hardware.camera.metadata@3.4", + "android.hardware.camera.metadata@3.5", + "android.hardware.camera.metadata@3.6", + "android.hardware.graphics.common@1.0", + "android.hidl.base@1.0", + ], + gen_java: false, +} diff --git a/camera/device/3.8/ICameraDevice.hal b/camera/device/3.8/ICameraDevice.hal new file mode 100644 index 0000000000..448f176220 --- /dev/null +++ b/camera/device/3.8/ICameraDevice.hal @@ -0,0 +1,32 @@ +/* + * Copyright (C) 2021 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package android.hardware.camera.device@3.8; + +import @3.7::ICameraDevice; + +/** + * Camera device interface + * + * Supports the android.hardware.Camera API, and the android.hardware.camera2 + * API at LIMITED or better hardware level. + * + * ICameraDevice.open() must return @3.2::ICameraDeviceSession, + * @3.5::ICameraDeviceSession, @3.6::ICameraDeviceSession, or + * @3.7::ICameraDeviceSession. + */ +interface ICameraDevice extends @3.7::ICameraDevice { +}; diff --git a/camera/device/3.8/ICameraDeviceCallback.hal b/camera/device/3.8/ICameraDeviceCallback.hal new file mode 100644 index 0000000000..de0775d59f --- /dev/null +++ b/camera/device/3.8/ICameraDeviceCallback.hal @@ -0,0 +1,34 @@ +/* + * Copyright (C) 2021 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package android.hardware.camera.device@3.8; + +import @3.5::ICameraDeviceCallback; + +/** + * Callback methods for the HAL to call into the framework. + */ +interface ICameraDeviceCallback extends @3.5::ICameraDeviceCallback { + /** + * Identical to @3.5::ICameraDeviceCallback.notify, except that it takes a + * list of @3.8::NotifyMsg which contain readout timestamp in addition + * to exposure start timestamp for shutter. + * + * The readout timestamp is used for the framework to re-time the viewfinder + * frames targeted for SurfaceView so that preview jitter can be reduced. + */ + notify_3_8(vec msgs); +}; diff --git a/camera/device/3.8/types.hal b/camera/device/3.8/types.hal new file mode 100644 index 0000000000..6daa0e13b7 --- /dev/null +++ b/camera/device/3.8/types.hal @@ -0,0 +1,68 @@ +/* + * Copyright (C) 2021 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package android.hardware.camera.device@3.8; + +import @3.2::ErrorMsg; +import @3.2::MsgType; +import @3.2::ShutterMsg; + +/** + * ShutterMsg: + * + * Message contents for MsgType::SHUTTER + * + * This version extends the @3.2 ShutterMsg with the readout timestamp. + */ +struct ShutterMsg { + /** + * The definition of ShutterMsg from prior version. + */ + @3.2::ShutterMsg v3_2; + + /** + * Timestamp for the capture readout. This must be in the same time domain + * as v3_2.timestamp, and the value must be v3_2.timestamp + exposureTime + * for a rolling shutter sensor. + */ + uint64_t readoutTimestamp; +}; + +/** + * NotifyMsg: + * + * The message structure sent to ICameraDevice3Callback::notify() + * + * This version extends the @3.2 NotifyMsg with the @3.8 version of ShutterMsg. + */ +struct NotifyMsg { + /** + * The message type. + */ + @3.2::MsgType type; + + union Message { + /** + * Error message contents. Valid if type is MsgType::ERROR + */ + @3.2::ErrorMsg error; + + /** + * Shutter message contents. Valid if type is MsgType::SHUTTER + */ + ShutterMsg shutter; + } msg; +}; diff --git a/camera/provider/2.4/vts/functional/Android.bp b/camera/provider/2.4/vts/functional/Android.bp index 8886ee1e8c..2c141ee1d6 100644 --- a/camera/provider/2.4/vts/functional/Android.bp +++ b/camera/provider/2.4/vts/functional/Android.bp @@ -49,6 +49,7 @@ cc_test { "android.hardware.camera.device@3.5", "android.hardware.camera.device@3.6", "android.hardware.camera.device@3.7", + "android.hardware.camera.device@3.8", "android.hardware.camera.metadata@3.4", "android.hardware.camera.provider@2.4", "android.hardware.camera.provider@2.5", diff --git a/camera/provider/2.4/vts/functional/VtsHalCameraProviderV2_4TargetTest.cpp b/camera/provider/2.4/vts/functional/VtsHalCameraProviderV2_4TargetTest.cpp index d02547c81e..ff8cd49e26 100644 --- a/camera/provider/2.4/vts/functional/VtsHalCameraProviderV2_4TargetTest.cpp +++ b/camera/provider/2.4/vts/functional/VtsHalCameraProviderV2_4TargetTest.cpp @@ -43,6 +43,7 @@ #include #include #include +#include #include #include #include @@ -194,6 +195,7 @@ enum SystemCameraKind { namespace { // "device@/legacy/" const char *kDeviceNameRE = "device@([0-9]+\\.[0-9]+)/%s/(.+)"; + const int CAMERA_DEVICE_API_VERSION_3_8 = 0x308; const int CAMERA_DEVICE_API_VERSION_3_7 = 0x307; const int CAMERA_DEVICE_API_VERSION_3_6 = 0x306; const int CAMERA_DEVICE_API_VERSION_3_5 = 0x305; @@ -201,6 +203,7 @@ namespace { const int CAMERA_DEVICE_API_VERSION_3_3 = 0x303; const int CAMERA_DEVICE_API_VERSION_3_2 = 0x302; const int CAMERA_DEVICE_API_VERSION_1_0 = 0x100; + const char *kHAL3_8 = "3.8"; const char *kHAL3_7 = "3.7"; const char *kHAL3_6 = "3.6"; const char *kHAL3_5 = "3.5"; @@ -238,7 +241,9 @@ namespace { return -1; } - if (version.compare(kHAL3_7) == 0) { + if (version.compare(kHAL3_8) == 0) { + return CAMERA_DEVICE_API_VERSION_3_8; + } else if (version.compare(kHAL3_7) == 0) { return CAMERA_DEVICE_API_VERSION_3_7; } else if (version.compare(kHAL3_6) == 0) { return CAMERA_DEVICE_API_VERSION_3_6; @@ -638,7 +643,7 @@ public: } }; - struct DeviceCb : public V3_5::ICameraDeviceCallback { + struct DeviceCb : public V3_8::ICameraDeviceCallback { DeviceCb(CameraHidlTest *parent, int deviceVersion, const camera_metadata_t *staticMeta) : mParent(parent), mDeviceVersion(deviceVersion) { mStaticMetadata = staticMeta; @@ -648,6 +653,7 @@ public: const hidl_vec& results) override; Return processCaptureResult(const hidl_vec& results) override; Return notify(const hidl_vec& msgs) override; + Return notify_3_8(const hidl_vec& msgs) override; Return requestStreamBuffers( const hidl_vec& bufReqs, @@ -663,6 +669,8 @@ public: private: bool processCaptureResultLocked(const CaptureResult& results, hidl_vec physicalCameraMetadata); + Return notifyHelper(const hidl_vec& msgs, + const std::vector>& readoutTimestamps); CameraHidlTest *mParent; // Parent object int mDeviceVersion; @@ -956,6 +964,9 @@ protected: // Set by notify() SHUTTER call. nsecs_t shutterTimestamp; + bool shutterReadoutTimestampValid; + nsecs_t shutterReadoutTimestamp; + bool errorCodeValid; ErrorCode errorCode; @@ -1001,6 +1012,8 @@ protected: InFlightRequest() : shutterTimestamp(0), + shutterReadoutTimestampValid(false), + shutterReadoutTimestamp(0), errorCodeValid(false), errorCode(ErrorCode::ERROR_BUFFER), usePartialResult(false), @@ -1018,6 +1031,8 @@ protected: bool partialResults, uint32_t partialCount, std::shared_ptr queue = nullptr) : shutterTimestamp(0), + shutterReadoutTimestampValid(false), + shutterReadoutTimestamp(0), errorCodeValid(false), errorCode(ErrorCode::ERROR_BUFFER), usePartialResult(partialResults), @@ -1036,6 +1051,8 @@ protected: const std::unordered_set& extraPhysicalResult, std::shared_ptr queue = nullptr) : shutterTimestamp(0), + shutterReadoutTimestampValid(false), + shutterReadoutTimestamp(0), errorCodeValid(false), errorCode(ErrorCode::ERROR_BUFFER), usePartialResult(partialResults), @@ -1462,8 +1479,46 @@ void CameraHidlTest::DeviceCb::waitForBuffersReturned() { } } +Return CameraHidlTest::DeviceCb::notify_3_8( + const hidl_vec& msgs) { + hidl_vec msgs3_2; + std::vector> readoutTimestamps; + + nsecs_t count = msgs.size(); + msgs3_2.resize(count); + readoutTimestamps.resize(count); + + for (size_t i = 0; i < count; i++) { + msgs3_2[i].type = msgs[i].type; + switch (msgs[i].type) { + case MsgType::ERROR: + msgs3_2[i].msg.error = msgs[i].msg.error; + readoutTimestamps[i] = {false, 0}; + break; + case MsgType::SHUTTER: + msgs3_2[i].msg.shutter = msgs[i].msg.shutter.v3_2; + readoutTimestamps[i] = {true, msgs[i].msg.shutter.readoutTimestamp}; + break; + } + } + + return notifyHelper(msgs3_2, readoutTimestamps); +} + Return CameraHidlTest::DeviceCb::notify( const hidl_vec& messages) { + std::vector> readoutTimestamps; + readoutTimestamps.resize(messages.size()); + for (size_t i = 0; i < messages.size(); i++) { + readoutTimestamps[i] = {false, 0}; + } + + return notifyHelper(messages, readoutTimestamps); +} + +Return CameraHidlTest::DeviceCb::notifyHelper( + const hidl_vec& messages, + const std::vector>& readoutTimestamps) { std::lock_guard l(mParent->mLock); for (size_t i = 0; i < messages.size(); i++) { @@ -1526,6 +1581,8 @@ Return CameraHidlTest::DeviceCb::notify( } InFlightRequest *r = mParent->mInflightMap.editValueAt(idx); r->shutterTimestamp = messages[i].msg.shutter.timestamp; + r->shutterReadoutTimestampValid = readoutTimestamps[i].first; + r->shutterReadoutTimestamp = readoutTimestamps[i].second; } break; default: @@ -1940,6 +1997,7 @@ TEST_P(CameraHidlTest, getCameraDeviceInterface) { for (const auto& name : cameraDeviceNames) { int deviceVersion = getCameraDeviceVersion(name, mProviderType); switch (deviceVersion) { + case CAMERA_DEVICE_API_VERSION_3_8: case CAMERA_DEVICE_API_VERSION_3_7: case CAMERA_DEVICE_API_VERSION_3_6: case CAMERA_DEVICE_API_VERSION_3_5: @@ -1984,6 +2042,7 @@ TEST_P(CameraHidlTest, getResourceCost) { for (const auto& name : cameraDeviceNames) { int deviceVersion = getCameraDeviceVersion(name, mProviderType); switch (deviceVersion) { + case CAMERA_DEVICE_API_VERSION_3_8: case CAMERA_DEVICE_API_VERSION_3_7: case CAMERA_DEVICE_API_VERSION_3_6: case CAMERA_DEVICE_API_VERSION_3_5: @@ -2725,6 +2784,7 @@ TEST_P(CameraHidlTest, systemCameraTest) { for (const auto& name : cameraDeviceNames) { int deviceVersion = getCameraDeviceVersion(name, mProviderType); switch (deviceVersion) { + case CAMERA_DEVICE_API_VERSION_3_8: case CAMERA_DEVICE_API_VERSION_3_7: case CAMERA_DEVICE_API_VERSION_3_6: case CAMERA_DEVICE_API_VERSION_3_5: @@ -2812,6 +2872,7 @@ TEST_P(CameraHidlTest, getCameraCharacteristics) { for (const auto& name : cameraDeviceNames) { int deviceVersion = getCameraDeviceVersion(name, mProviderType); switch (deviceVersion) { + case CAMERA_DEVICE_API_VERSION_3_8: case CAMERA_DEVICE_API_VERSION_3_7: case CAMERA_DEVICE_API_VERSION_3_6: case CAMERA_DEVICE_API_VERSION_3_5: @@ -2893,6 +2954,7 @@ TEST_P(CameraHidlTest, setTorchMode) { for (const auto& name : cameraDeviceNames) { int deviceVersion = getCameraDeviceVersion(name, mProviderType); switch (deviceVersion) { + case CAMERA_DEVICE_API_VERSION_3_8: case CAMERA_DEVICE_API_VERSION_3_7: case CAMERA_DEVICE_API_VERSION_3_6: case CAMERA_DEVICE_API_VERSION_3_5: @@ -3021,6 +3083,7 @@ TEST_P(CameraHidlTest, dumpState) { for (const auto& name : cameraDeviceNames) { int deviceVersion = getCameraDeviceVersion(name, mProviderType); switch (deviceVersion) { + case CAMERA_DEVICE_API_VERSION_3_8: case CAMERA_DEVICE_API_VERSION_3_7: case CAMERA_DEVICE_API_VERSION_3_6: case CAMERA_DEVICE_API_VERSION_3_5: @@ -3088,6 +3151,7 @@ TEST_P(CameraHidlTest, openClose) { for (const auto& name : cameraDeviceNames) { int deviceVersion = getCameraDeviceVersion(name, mProviderType); switch (deviceVersion) { + case CAMERA_DEVICE_API_VERSION_3_8: case CAMERA_DEVICE_API_VERSION_3_7: case CAMERA_DEVICE_API_VERSION_3_6: case CAMERA_DEVICE_API_VERSION_3_5: @@ -3124,7 +3188,7 @@ TEST_P(CameraHidlTest, openClose) { castSession(session, deviceVersion, &sessionV3_3, &sessionV3_4, &sessionV3_5, &sessionV3_6, &sessionV3_7); - if (deviceVersion == CAMERA_DEVICE_API_VERSION_3_7) { + if (deviceVersion >= CAMERA_DEVICE_API_VERSION_3_7) { ASSERT_TRUE(sessionV3_7.get() != nullptr); } else if (deviceVersion == CAMERA_DEVICE_API_VERSION_3_6) { ASSERT_TRUE(sessionV3_6.get() != nullptr); @@ -3190,6 +3254,7 @@ TEST_P(CameraHidlTest, constructDefaultRequestSettings) { for (const auto& name : cameraDeviceNames) { int deviceVersion = getCameraDeviceVersion(name, mProviderType); switch (deviceVersion) { + case CAMERA_DEVICE_API_VERSION_3_8: case CAMERA_DEVICE_API_VERSION_3_7: case CAMERA_DEVICE_API_VERSION_3_6: case CAMERA_DEVICE_API_VERSION_3_5: @@ -4730,6 +4795,19 @@ void CameraHidlTest::processCaptureRequestInternal(uint64_t bufferUsage, ASSERT_NE(inflightReq.resultOutputBuffers.size(), 0u); ASSERT_EQ(testStream.id, inflightReq.resultOutputBuffers[0].streamId); + // For camera device 3.8 or newer, shutterReadoutTimestamp must be + // available, and it must be shutterTimestamp + exposureTime. + if (deviceVersion >= CAMERA_DEVICE_API_VERSION_3_8) { + ASSERT_TRUE(inflightReq.shutterReadoutTimestampValid); + ASSERT_FALSE(inflightReq.collectedResult.isEmpty()); + if (inflightReq.collectedResult.exists(ANDROID_SENSOR_EXPOSURE_TIME)) { + camera_metadata_entry_t exposureTimeResult = inflightReq.collectedResult.find( + ANDROID_SENSOR_EXPOSURE_TIME); + ASSERT_EQ(inflightReq.shutterReadoutTimestamp - inflightReq.shutterTimestamp, + exposureTimeResult.data.i64[0]); + } + } + request.frameNumber++; // Empty settings should be supported after the first call // for repeating requests. @@ -6222,6 +6300,7 @@ TEST_P(CameraHidlTest, grfSMultiCameraTest) { std::string cameraId; int deviceVersion = getCameraDeviceVersionAndId(name, mProviderType, &cameraId); switch (deviceVersion) { + case CAMERA_DEVICE_API_VERSION_3_8: case CAMERA_DEVICE_API_VERSION_3_7: case CAMERA_DEVICE_API_VERSION_3_6: case CAMERA_DEVICE_API_VERSION_3_5: @@ -7651,6 +7730,7 @@ void CameraHidlTest::castDevice(const sp& device, ASSERT_NE(nullptr, device3_7); switch (deviceVersion) { + case CAMERA_DEVICE_API_VERSION_3_8: case CAMERA_DEVICE_API_VERSION_3_7: { auto castResult = device::V3_7::ICameraDevice::castFrom(device); ASSERT_TRUE(castResult.isOk()); @@ -7707,6 +7787,7 @@ void CameraHidlTest::castSession(const sp &session, int32_ ASSERT_NE(nullptr, session3_7); switch (deviceVersion) { + case CAMERA_DEVICE_API_VERSION_3_8: case CAMERA_DEVICE_API_VERSION_3_7: { auto castResult = device::V3_7::ICameraDeviceSession::castFrom(session); ASSERT_TRUE(castResult.isOk());