/* * Copyright (C) 2012 The Android Open Source Project * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #define LOG_TAG "Camera2-ZslProcessor" #define ATRACE_TAG ATRACE_TAG_CAMERA //#define LOG_NDEBUG 0 //#define LOG_NNDEBUG 0 #ifdef LOG_NNDEBUG #define ALOGVV(...) ALOGV(__VA_ARGS__) #else #define ALOGVV(...) ((void)0) #endif #include <inttypes.h> #include <utils/Log.h> #include <utils/Trace.h> #include <gui/Surface.h> #include "common/CameraDeviceBase.h" #include "api1/Camera2Client.h" #include "api1/client2/CaptureSequencer.h" #include "api1/client2/ZslProcessor.h" namespace android { namespace camera2 { ZslProcessor::ZslProcessor( sp<Camera2Client> client, wp<CaptureSequencer> sequencer): Thread(false), mState(RUNNING), mClient(client), mDevice(client->getCameraDevice()), mSequencer(sequencer), mId(client->getCameraId()), mDeleted(false), mZslBufferAvailable(false), mZslStreamId(NO_STREAM), mZslReprocessStreamId(NO_STREAM), mFrameListHead(0), mZslQueueHead(0), mZslQueueTail(0) { mZslQueue.insertAt(0, kZslBufferDepth); mFrameList.insertAt(0, kFrameListDepth); sp<CaptureSequencer> captureSequencer = mSequencer.promote(); if (captureSequencer != 0) captureSequencer->setZslProcessor(this); } ZslProcessor::~ZslProcessor() { ALOGV("%s: Exit", __FUNCTION__); disconnect(); } void ZslProcessor::onFrameAvailable(const BufferItem& /*item*/) { Mutex::Autolock l(mInputMutex); if (!mZslBufferAvailable) { mZslBufferAvailable = true; mZslBufferAvailableSignal.signal(); } } void ZslProcessor::onResultAvailable(const CaptureResult &result) { ATRACE_CALL(); ALOGV("%s:", __FUNCTION__); Mutex::Autolock l(mInputMutex); camera_metadata_ro_entry_t entry; entry = result.mMetadata.find(ANDROID_SENSOR_TIMESTAMP); nsecs_t timestamp = entry.data.i64[0]; (void)timestamp; ALOGVV("Got preview frame for timestamp %" PRId64, timestamp); if (mState != RUNNING) return; mFrameList.editItemAt(mFrameListHead) = result.mMetadata; mFrameListHead = (mFrameListHead + 1) % kFrameListDepth; findMatchesLocked(); } void ZslProcessor::onBufferReleased(buffer_handle_t *handle) { Mutex::Autolock l(mInputMutex); // Verify that the buffer is in our queue size_t i = 0; for (; i < mZslQueue.size(); i++) { if (&(mZslQueue[i].buffer.mGraphicBuffer->handle) == handle) break; } if (i == mZslQueue.size()) { ALOGW("%s: Released buffer %p not found in queue", __FUNCTION__, handle); } // Erase entire ZSL queue since we've now completed the capture and preview // is stopped. clearZslQueueLocked(); mState = RUNNING; } status_t ZslProcessor::updateStream(const Parameters ¶ms) { ATRACE_CALL(); ALOGV("%s: Configuring ZSL streams", __FUNCTION__); status_t res; Mutex::Autolock l(mInputMutex); sp<Camera2Client> client = mClient.promote(); if (client == 0) { ALOGE("%s: Camera %d: Client does not exist", __FUNCTION__, mId); return INVALID_OPERATION; } sp<CameraDeviceBase> device = mDevice.promote(); if (device == 0) { ALOGE("%s: Camera %d: Device does not exist", __FUNCTION__, mId); return INVALID_OPERATION; } if (mZslConsumer == 0) { // Create CPU buffer queue endpoint sp<IGraphicBufferProducer> producer; sp<IGraphicBufferConsumer> consumer; BufferQueue::createBufferQueue(&producer, &consumer); mZslConsumer = new BufferItemConsumer(consumer, GRALLOC_USAGE_HW_CAMERA_ZSL, kZslBufferDepth); mZslConsumer->setFrameAvailableListener(this); mZslConsumer->setName(String8("Camera2-ZslConsumer")); mZslWindow = new Surface(producer); } if (mZslStreamId != NO_STREAM) { // Check if stream parameters have to change uint32_t currentWidth, currentHeight; res = device->getStreamInfo(mZslStreamId, ¤tWidth, ¤tHeight, 0, 0); if (res != OK) { ALOGE("%s: Camera %d: Error querying capture output stream info: " "%s (%d)", __FUNCTION__, mId, strerror(-res), res); return res; } if (currentWidth != (uint32_t)params.fastInfo.arrayWidth || currentHeight != (uint32_t)params.fastInfo.arrayHeight) { res = device->deleteReprocessStream(mZslReprocessStreamId); if (res != OK) { ALOGE("%s: Camera %d: Unable to delete old reprocess stream " "for ZSL: %s (%d)", __FUNCTION__, mId, strerror(-res), res); return res; } ALOGV("%s: Camera %d: Deleting stream %d since the buffer dimensions changed", __FUNCTION__, mId, mZslStreamId); res = device->deleteStream(mZslStreamId); if (res != OK) { ALOGE("%s: Camera %d: Unable to delete old output stream " "for ZSL: %s (%d)", __FUNCTION__, mId, strerror(-res), res); return res; } mZslStreamId = NO_STREAM; } } mDeleted = false; if (mZslStreamId == NO_STREAM) { // Create stream for HAL production // TODO: Sort out better way to select resolution for ZSL int streamType = params.quirks.useZslFormat ? (int)CAMERA2_HAL_PIXEL_FORMAT_ZSL : (int)HAL_PIXEL_FORMAT_IMPLEMENTATION_DEFINED; res = device->createStream(mZslWindow, params.fastInfo.arrayWidth, params.fastInfo.arrayHeight, streamType, HAL_DATASPACE_UNKNOWN, CAMERA3_STREAM_ROTATION_0, &mZslStreamId); if (res != OK) { ALOGE("%s: Camera %d: Can't create output stream for ZSL: " "%s (%d)", __FUNCTION__, mId, strerror(-res), res); return res; } res = device->createReprocessStreamFromStream(mZslStreamId, &mZslReprocessStreamId); if (res != OK) { ALOGE("%s: Camera %d: Can't create reprocess stream for ZSL: " "%s (%d)", __FUNCTION__, mId, strerror(-res), res); return res; } } client->registerFrameListener(Camera2Client::kPreviewRequestIdStart, Camera2Client::kPreviewRequestIdEnd, this, /*sendPartials*/false); return OK; } status_t ZslProcessor::deleteStream() { ATRACE_CALL(); Mutex::Autolock l(mInputMutex); // WAR(b/15408128): do not delete stream unless client is being disconnected. mDeleted = true; return OK; } status_t ZslProcessor::disconnect() { ATRACE_CALL(); status_t res; Mutex::Autolock l(mInputMutex); if (mZslStreamId != NO_STREAM) { sp<CameraDeviceBase> device = mDevice.promote(); if (device == 0) { ALOGE("%s: Camera %d: Device does not exist", __FUNCTION__, mId); return INVALID_OPERATION; } clearZslQueueLocked(); res = device->deleteReprocessStream(mZslReprocessStreamId); if (res != OK) { ALOGE("%s: Camera %d: Cannot delete ZSL reprocessing stream %d: " "%s (%d)", __FUNCTION__, mId, mZslReprocessStreamId, strerror(-res), res); return res; } mZslReprocessStreamId = NO_STREAM; res = device->deleteStream(mZslStreamId); if (res != OK) { ALOGE("%s: Camera %d: Cannot delete ZSL output stream %d: " "%s (%d)", __FUNCTION__, mId, mZslStreamId, strerror(-res), res); return res; } mZslWindow.clear(); mZslConsumer.clear(); mZslStreamId = NO_STREAM; } return OK; } int ZslProcessor::getStreamId() const { Mutex::Autolock l(mInputMutex); return mZslStreamId; } status_t ZslProcessor::pushToReprocess(int32_t requestId) { ALOGV("%s: Send in reprocess request with id %d", __FUNCTION__, requestId); Mutex::Autolock l(mInputMutex); status_t res; sp<Camera2Client> client = mClient.promote(); if (client == 0) { ALOGE("%s: Camera %d: Client does not exist", __FUNCTION__, mId); return INVALID_OPERATION; } IF_ALOGV() { dumpZslQueue(-1); } if (mZslQueueTail != mZslQueueHead) { CameraMetadata request; size_t index = mZslQueueTail; while (index != mZslQueueHead) { if (!mZslQueue[index].frame.isEmpty()) { request = mZslQueue[index].frame; break; } index = (index + 1) % kZslBufferDepth; } if (index == mZslQueueHead) { ALOGV("%s: ZSL queue has no valid frames to send yet.", __FUNCTION__); return NOT_ENOUGH_DATA; } // Verify that the frame is reasonable for reprocessing camera_metadata_entry_t entry; entry = request.find(ANDROID_CONTROL_AE_STATE); if (entry.count == 0) { ALOGE("%s: ZSL queue frame has no AE state field!", __FUNCTION__); return BAD_VALUE; } if (entry.data.u8[0] != ANDROID_CONTROL_AE_STATE_CONVERGED && entry.data.u8[0] != ANDROID_CONTROL_AE_STATE_LOCKED) { ALOGV("%s: ZSL queue frame AE state is %d, need full capture", __FUNCTION__, entry.data.u8[0]); return NOT_ENOUGH_DATA; } buffer_handle_t *handle = &(mZslQueue[index].buffer.mGraphicBuffer->handle); uint8_t requestType = ANDROID_REQUEST_TYPE_REPROCESS; res = request.update(ANDROID_REQUEST_TYPE, &requestType, 1); int32_t inputStreams[1] = { mZslReprocessStreamId }; if (res == OK) request.update(ANDROID_REQUEST_INPUT_STREAMS, inputStreams, 1); int32_t outputStreams[1] = { client->getCaptureStreamId() }; if (res == OK) request.update(ANDROID_REQUEST_OUTPUT_STREAMS, outputStreams, 1); res = request.update(ANDROID_REQUEST_ID, &requestId, 1); if (res != OK ) { ALOGE("%s: Unable to update frame to a reprocess request", __FUNCTION__); return INVALID_OPERATION; } res = client->stopStream(); if (res != OK) { ALOGE("%s: Camera %d: Unable to stop preview for ZSL capture: " "%s (%d)", __FUNCTION__, mId, strerror(-res), res); return INVALID_OPERATION; } // TODO: have push-and-clear be atomic res = client->getCameraDevice()->pushReprocessBuffer(mZslReprocessStreamId, handle, this); if (res != OK) { ALOGE("%s: Unable to push buffer for reprocessing: %s (%d)", __FUNCTION__, strerror(-res), res); return res; } // Update JPEG settings { SharedParameters::Lock l(client->getParameters()); res = l.mParameters.updateRequestJpeg(&request); if (res != OK) { ALOGE("%s: Camera %d: Unable to update JPEG entries of ZSL " "capture request: %s (%d)", __FUNCTION__, mId, strerror(-res), res); return res; } } mLatestCapturedRequest = request; res = client->getCameraDevice()->capture(request); if (res != OK ) { ALOGE("%s: Unable to send ZSL reprocess request to capture: %s (%d)", __FUNCTION__, strerror(-res), res); return res; } mState = LOCKED; } else { ALOGV("%s: No ZSL buffers yet", __FUNCTION__); return NOT_ENOUGH_DATA; } return OK; } status_t ZslProcessor::clearZslQueue() { Mutex::Autolock l(mInputMutex); // If in middle of capture, can't clear out queue if (mState == LOCKED) return OK; return clearZslQueueLocked(); } status_t ZslProcessor::clearZslQueueLocked() { for (size_t i = 0; i < mZslQueue.size(); i++) { if (mZslQueue[i].buffer.mTimestamp != 0) { mZslConsumer->releaseBuffer(mZslQueue[i].buffer); } mZslQueue.replaceAt(i); } mZslQueueHead = 0; mZslQueueTail = 0; return OK; } void ZslProcessor::dump(int fd, const Vector<String16>& /*args*/) const { Mutex::Autolock l(mInputMutex); if (!mLatestCapturedRequest.isEmpty()) { String8 result(" Latest ZSL capture request:\n"); write(fd, result.string(), result.size()); mLatestCapturedRequest.dump(fd, 2, 6); } else { String8 result(" Latest ZSL capture request: none yet\n"); write(fd, result.string(), result.size()); } dumpZslQueue(fd); } bool ZslProcessor::threadLoop() { status_t res; { Mutex::Autolock l(mInputMutex); while (!mZslBufferAvailable) { res = mZslBufferAvailableSignal.waitRelative(mInputMutex, kWaitDuration); if (res == TIMED_OUT) return true; } mZslBufferAvailable = false; } do { res = processNewZslBuffer(); } while (res == OK); return true; } status_t ZslProcessor::processNewZslBuffer() { ATRACE_CALL(); status_t res; sp<BufferItemConsumer> zslConsumer; { Mutex::Autolock l(mInputMutex); if (mZslConsumer == 0) return OK; zslConsumer = mZslConsumer; } ALOGVV("Trying to get next buffer"); BufferItem item; res = zslConsumer->acquireBuffer(&item, 0); if (res != OK) { if (res != BufferItemConsumer::NO_BUFFER_AVAILABLE) { ALOGE("%s: Camera %d: Error receiving ZSL image buffer: " "%s (%d)", __FUNCTION__, mId, strerror(-res), res); } else { ALOGVV(" No buffer"); } return res; } Mutex::Autolock l(mInputMutex); if (mState == LOCKED) { ALOGVV("In capture, discarding new ZSL buffers"); zslConsumer->releaseBuffer(item); return OK; } ALOGVV("Got ZSL buffer: head: %d, tail: %d", mZslQueueHead, mZslQueueTail); if ( (mZslQueueHead + 1) % kZslBufferDepth == mZslQueueTail) { ALOGVV("Releasing oldest buffer"); zslConsumer->releaseBuffer(mZslQueue[mZslQueueTail].buffer); mZslQueue.replaceAt(mZslQueueTail); mZslQueueTail = (mZslQueueTail + 1) % kZslBufferDepth; } ZslPair &queueHead = mZslQueue.editItemAt(mZslQueueHead); queueHead.buffer = item; queueHead.frame.release(); mZslQueueHead = (mZslQueueHead + 1) % kZslBufferDepth; ALOGVV(" Acquired buffer, timestamp %" PRId64, queueHead.buffer.mTimestamp); findMatchesLocked(); return OK; } void ZslProcessor::findMatchesLocked() { ALOGVV("Scanning"); for (size_t i = 0; i < mZslQueue.size(); i++) { ZslPair &queueEntry = mZslQueue.editItemAt(i); nsecs_t bufferTimestamp = queueEntry.buffer.mTimestamp; IF_ALOGV() { camera_metadata_entry_t entry; nsecs_t frameTimestamp = 0; if (!queueEntry.frame.isEmpty()) { entry = queueEntry.frame.find(ANDROID_SENSOR_TIMESTAMP); frameTimestamp = entry.data.i64[0]; } ALOGVV(" %d: b: %" PRId64 "\tf: %" PRId64, i, bufferTimestamp, frameTimestamp ); } if (queueEntry.frame.isEmpty() && bufferTimestamp != 0) { // Have buffer, no matching frame. Look for one for (size_t j = 0; j < mFrameList.size(); j++) { bool match = false; CameraMetadata &frame = mFrameList.editItemAt(j); if (!frame.isEmpty()) { camera_metadata_entry_t entry; entry = frame.find(ANDROID_SENSOR_TIMESTAMP); if (entry.count == 0) { ALOGE("%s: Can't find timestamp in frame!", __FUNCTION__); continue; } nsecs_t frameTimestamp = entry.data.i64[0]; if (bufferTimestamp == frameTimestamp) { ALOGVV("%s: Found match %" PRId64, __FUNCTION__, frameTimestamp); match = true; } else { int64_t delta = abs(bufferTimestamp - frameTimestamp); if ( delta < 1000000) { ALOGVV("%s: Found close match %" PRId64 " (delta %" PRId64 ")", __FUNCTION__, bufferTimestamp, delta); match = true; } } } if (match) { queueEntry.frame.acquire(frame); break; } } } } } void ZslProcessor::dumpZslQueue(int fd) const { String8 header("ZSL queue contents:"); String8 indent(" "); ALOGV("%s", header.string()); if (fd != -1) { header = indent + header + "\n"; write(fd, header.string(), header.size()); } for (size_t i = 0; i < mZslQueue.size(); i++) { const ZslPair &queueEntry = mZslQueue[i]; nsecs_t bufferTimestamp = queueEntry.buffer.mTimestamp; camera_metadata_ro_entry_t entry; nsecs_t frameTimestamp = 0; int frameAeState = -1; if (!queueEntry.frame.isEmpty()) { entry = queueEntry.frame.find(ANDROID_SENSOR_TIMESTAMP); if (entry.count > 0) frameTimestamp = entry.data.i64[0]; entry = queueEntry.frame.find(ANDROID_CONTROL_AE_STATE); if (entry.count > 0) frameAeState = entry.data.u8[0]; } String8 result = String8::format(" %zu: b: %" PRId64 "\tf: %" PRId64 ", AE state: %d", i, bufferTimestamp, frameTimestamp, frameAeState); ALOGV("%s", result.string()); if (fd != -1) { result = indent + result + "\n"; write(fd, result.string(), result.size()); } } } }; // namespace camera2 }; // namespace android