/* * Copyright (C) 2012 The Android Open Source Project * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ //#define LOG_NDEBUG 0 //#define LOG_NNDEBUG 0 #define LOG_TAG "EmulatedCamera2_Sensor" #ifdef LOG_NNDEBUG #define ALOGVV(...) ALOGV(__VA_ARGS__) #else #define ALOGVV(...) ((void)0) #endif #include <utils/Log.h> #include <cmath> #include <cstdlib> #include "../EmulatedFakeCamera2.h" #include "Sensor.h" #include "guest/libs/platform_support/api_level_fixes.h" #include "system/camera_metadata.h" namespace android { // const nsecs_t Sensor::kExposureTimeRange[2] = // {1000L, 30000000000L} ; // 1 us - 30 sec // const nsecs_t Sensor::kFrameDurationRange[2] = // {33331760L, 30000000000L}; // ~1/30 s - 30 sec const nsecs_t Sensor::kExposureTimeRange[2] = {1000L, 300000000L}; // 1 us - 0.3 sec const nsecs_t Sensor::kFrameDurationRange[2] = { 33331760L, 300000000L}; // ~1/30 s - 0.3 sec const nsecs_t Sensor::kMinVerticalBlank = 10000L; const uint8_t Sensor::kColorFilterArrangement = ANDROID_SENSOR_INFO_COLOR_FILTER_ARRANGEMENT_RGGB; // Output image data characteristics const uint32_t Sensor::kMaxRawValue = 4000; const uint32_t Sensor::kBlackLevel = 1000; // Sensor sensitivity const float Sensor::kSaturationVoltage = 0.520f; const uint32_t Sensor::kSaturationElectrons = 2000; const float Sensor::kVoltsPerLuxSecond = 0.100f; const float Sensor::kElectronsPerLuxSecond = Sensor::kSaturationElectrons / Sensor::kSaturationVoltage * Sensor::kVoltsPerLuxSecond; const float Sensor::kBaseGainFactor = (float)Sensor::kMaxRawValue / Sensor::kSaturationElectrons; const float Sensor::kReadNoiseStddevBeforeGain = 1.177; // in electrons const float Sensor::kReadNoiseStddevAfterGain = 2.100; // in digital counts const float Sensor::kReadNoiseVarBeforeGain = Sensor::kReadNoiseStddevBeforeGain * Sensor::kReadNoiseStddevBeforeGain; const float Sensor::kReadNoiseVarAfterGain = Sensor::kReadNoiseStddevAfterGain * Sensor::kReadNoiseStddevAfterGain; const int32_t Sensor::kSensitivityRange[2] = {100, 1600}; const uint32_t Sensor::kDefaultSensitivity = 100; /** A few utility functions for math, normal distributions */ // Take advantage of IEEE floating-point format to calculate an approximate // square root. Accurate to within +-3.6% float sqrtf_approx(float r) { // Modifier is based on IEEE floating-point representation; the // manipulations boil down to finding approximate log2, dividing by two, and // then inverting the log2. A bias is added to make the relative error // symmetric about the real answer. const int32_t modifier = 0x1FBB4000; int32_t r_i = *(int32_t *)(&r); r_i = (r_i >> 1) + modifier; return *(float *)(&r_i); } Sensor::Sensor(uint32_t width, uint32_t height) : Thread(false), mResolution{width, height}, mActiveArray{0, 0, width, height}, mRowReadoutTime(kFrameDurationRange[0] / height), mGotVSync(false), mExposureTime(kFrameDurationRange[0] - kMinVerticalBlank), mFrameDuration(kFrameDurationRange[0]), mGainFactor(kDefaultSensitivity), mNextBuffers(NULL), mFrameNumber(0), mCapturedBuffers(NULL), mListener(NULL), mScene(width, height, kElectronsPerLuxSecond) { ALOGV("Sensor created with pixel array %d x %d", width, height); } Sensor::~Sensor() { shutDown(); } status_t Sensor::startUp() { ALOGV("%s: E", __FUNCTION__); int res; mCapturedBuffers = NULL; res = run("EmulatedFakeCamera2::Sensor", ANDROID_PRIORITY_URGENT_DISPLAY); if (res != OK) { ALOGE("Unable to start up sensor capture thread: %d", res); } return res; } status_t Sensor::shutDown() { ALOGV("%s: E", __FUNCTION__); int res; res = requestExitAndWait(); if (res != OK) { ALOGE("Unable to shut down sensor capture thread: %d", res); } return res; } Scene &Sensor::getScene() { return mScene; } void Sensor::setExposureTime(uint64_t ns) { Mutex::Autolock lock(mControlMutex); ALOGVV("Exposure set to %f", ns / 1000000.f); mExposureTime = ns; } void Sensor::setFrameDuration(uint64_t ns) { Mutex::Autolock lock(mControlMutex); ALOGVV("Frame duration set to %f", ns / 1000000.f); mFrameDuration = ns; } void Sensor::setSensitivity(uint32_t gain) { Mutex::Autolock lock(mControlMutex); ALOGVV("Gain set to %d", gain); mGainFactor = gain; } void Sensor::setDestinationBuffers(Buffers *buffers) { Mutex::Autolock lock(mControlMutex); mNextBuffers = buffers; } void Sensor::setFrameNumber(uint32_t frameNumber) { Mutex::Autolock lock(mControlMutex); mFrameNumber = frameNumber; } bool Sensor::waitForVSync(nsecs_t reltime) { int res; Mutex::Autolock lock(mControlMutex); mGotVSync = false; res = mVSync.waitRelative(mControlMutex, reltime); if (res != OK && res != TIMED_OUT) { ALOGE("%s: Error waiting for VSync signal: %d", __FUNCTION__, res); return false; } return mGotVSync; } bool Sensor::waitForNewFrame(nsecs_t reltime, nsecs_t *captureTime) { Mutex::Autolock lock(mReadoutMutex); if (mCapturedBuffers == NULL) { int res; res = mReadoutAvailable.waitRelative(mReadoutMutex, reltime); if (res == TIMED_OUT) { return false; } else if (res != OK || mCapturedBuffers == NULL) { ALOGE("Error waiting for sensor readout signal: %d", res); return false; } } mReadoutComplete.signal(); *captureTime = mCaptureTime; mCapturedBuffers = NULL; return true; } Sensor::SensorListener::~SensorListener() {} void Sensor::setSensorListener(SensorListener *listener) { Mutex::Autolock lock(mControlMutex); mListener = listener; } status_t Sensor::readyToRun() { ALOGV("Starting up sensor thread"); mStartupTime = systemTime(); mNextCaptureTime = 0; mNextCapturedBuffers = NULL; return OK; } bool Sensor::threadLoop() { /** * Sensor capture operation main loop. * * Stages are out-of-order relative to a single frame's processing, but * in-order in time. */ /** * Stage 1: Read in latest control parameters */ uint64_t exposureDuration; uint64_t frameDuration; uint32_t gain; Buffers *nextBuffers; uint32_t frameNumber; SensorListener *listener = NULL; { Mutex::Autolock lock(mControlMutex); exposureDuration = mExposureTime; frameDuration = mFrameDuration; gain = mGainFactor; nextBuffers = mNextBuffers; frameNumber = mFrameNumber; listener = mListener; // Don't reuse a buffer set mNextBuffers = NULL; // Signal VSync for start of readout ALOGVV("Sensor VSync"); mGotVSync = true; mVSync.signal(); } /** * Stage 3: Read out latest captured image */ Buffers *capturedBuffers = NULL; nsecs_t captureTime = 0; nsecs_t startRealTime = systemTime(); // Stagefright cares about system time for timestamps, so base simulated // time on that. nsecs_t simulatedTime = startRealTime; nsecs_t frameEndRealTime = startRealTime + frameDuration; if (mNextCapturedBuffers != NULL) { ALOGVV("Sensor starting readout"); // Pretend we're doing readout now; will signal once enough time has elapsed capturedBuffers = mNextCapturedBuffers; captureTime = mNextCaptureTime; } simulatedTime += mRowReadoutTime + kMinVerticalBlank; // TODO: Move this signal to another thread to simulate readout // time properly if (capturedBuffers != NULL) { ALOGVV("Sensor readout complete"); Mutex::Autolock lock(mReadoutMutex); if (mCapturedBuffers != NULL) { ALOGV("Waiting for readout thread to catch up!"); mReadoutComplete.wait(mReadoutMutex); } mCapturedBuffers = capturedBuffers; mCaptureTime = captureTime; mReadoutAvailable.signal(); capturedBuffers = NULL; } /** * Stage 2: Capture new image */ mNextCaptureTime = simulatedTime; mNextCapturedBuffers = nextBuffers; if (mNextCapturedBuffers != NULL) { if (listener != NULL) { listener->onSensorEvent(frameNumber, SensorListener::EXPOSURE_START, mNextCaptureTime); } ALOGVV("Starting next capture: Exposure: %f ms, gain: %d", (float)exposureDuration / 1e6, gain); mScene.setExposureDuration((float)exposureDuration / 1e9); mScene.calculateScene(mNextCaptureTime); // Might be adding more buffers, so size isn't constant for (size_t i = 0; i < mNextCapturedBuffers->size(); i++) { const StreamBuffer &b = (*mNextCapturedBuffers)[i]; ALOGVV( "Sensor capturing buffer %d: stream %d," " %d x %d, format %x, stride %d, buf %p, img %p", i, b.streamId, b.width, b.height, b.format, b.stride, b.buffer, b.img); switch (b.format) { #if VSOC_PLATFORM_SDK_AFTER(K) case HAL_PIXEL_FORMAT_RAW16: captureRaw(b.img, gain, b.stride); break; #endif case HAL_PIXEL_FORMAT_RGB_888: captureRGB(b.img, gain, b.stride); break; case HAL_PIXEL_FORMAT_RGBA_8888: captureRGBA(b.img, gain, b.stride); break; case HAL_PIXEL_FORMAT_BLOB: #if defined HAL_DATASPACE_DEPTH if (b.dataSpace != HAL_DATASPACE_DEPTH) { #endif // Add auxillary buffer of the right size // Assumes only one BLOB (JPEG) buffer in // mNextCapturedBuffers StreamBuffer bAux; bAux.streamId = 0; bAux.width = b.width; bAux.height = b.height; bAux.format = HAL_PIXEL_FORMAT_RGB_888; bAux.stride = b.width; bAux.buffer = NULL; // TODO: Reuse these bAux.img = new uint8_t[b.width * b.height * 3]; mNextCapturedBuffers->push_back(bAux); #if defined HAL_DATASPACE_DEPTH } else { captureDepthCloud(b.img); } #endif break; case HAL_PIXEL_FORMAT_YCrCb_420_SP: case HAL_PIXEL_FORMAT_YCbCr_420_888: captureNV21(b.img, gain, b.stride); break; case HAL_PIXEL_FORMAT_YV12: // TODO: ALOGE("%s: Format %x is TODO", __FUNCTION__, b.format); break; case HAL_PIXEL_FORMAT_Y16: captureDepth(b.img, gain, b.stride); break; default: ALOGE("%s: Unknown format %x, no output", __FUNCTION__, b.format); break; } } } ALOGVV("Sensor vertical blanking interval"); nsecs_t workDoneRealTime = systemTime(); const nsecs_t timeAccuracy = 2e6; // 2 ms of imprecision is ok if (workDoneRealTime < frameEndRealTime - timeAccuracy) { timespec t; t.tv_sec = (frameEndRealTime - workDoneRealTime) / 1000000000L; t.tv_nsec = (frameEndRealTime - workDoneRealTime) % 1000000000L; int ret; do { ret = nanosleep(&t, &t); } while (ret != 0); } nsecs_t endRealTime __unused = systemTime(); ALOGVV("Frame cycle took %d ms, target %d ms", (int)((endRealTime - startRealTime) / 1000000), (int)(frameDuration / 1000000)); return true; }; void Sensor::captureRaw(uint8_t *img, uint32_t gain, uint32_t stride) { float totalGain = gain / 100.0 * kBaseGainFactor; float noiseVarGain = totalGain * totalGain; float readNoiseVar = kReadNoiseVarBeforeGain * noiseVarGain + kReadNoiseVarAfterGain; int bayerSelect[4] = {Scene::R, Scene::Gr, Scene::Gb, Scene::B}; // RGGB mScene.setReadoutPixel(0, 0); for (unsigned int y = 0; y < mResolution[1]; y++) { int *bayerRow = bayerSelect + (y & 0x1) * 2; uint16_t *px = (uint16_t *)img + y * stride; for (unsigned int x = 0; x < mResolution[0]; x++) { uint32_t electronCount; electronCount = mScene.getPixelElectrons()[bayerRow[x & 0x1]]; // TODO: Better pixel saturation curve? electronCount = (electronCount < kSaturationElectrons) ? electronCount : kSaturationElectrons; // TODO: Better A/D saturation curve? uint16_t rawCount = electronCount * totalGain; rawCount = (rawCount < kMaxRawValue) ? rawCount : kMaxRawValue; // Calculate noise value // TODO: Use more-correct Gaussian instead of uniform noise float photonNoiseVar = electronCount * noiseVarGain; float noiseStddev = sqrtf_approx(readNoiseVar + photonNoiseVar); // Scaled to roughly match gaussian/uniform noise stddev float noiseSample = std::rand() * (2.5 / (1.0 + RAND_MAX)) - 1.25; rawCount += kBlackLevel; rawCount += noiseStddev * noiseSample; *px++ = rawCount; } // TODO: Handle this better // simulatedTime += mRowReadoutTime; } ALOGVV("Raw sensor image captured"); } void Sensor::captureRGBA(uint8_t *img, uint32_t gain, uint32_t stride) { float totalGain = gain / 100.0 * kBaseGainFactor; // In fixed-point math, calculate total scaling from electrons to 8bpp int scale64x = 64 * totalGain * 255 / kMaxRawValue; uint32_t inc = ceil((float)mResolution[0] / stride); for (unsigned int y = 0, outY = 0; y < mResolution[1]; y += inc, outY++) { uint8_t *px = img + outY * stride * 4; mScene.setReadoutPixel(0, y); for (unsigned int x = 0; x < mResolution[0]; x += inc) { uint32_t rCount, gCount, bCount; // TODO: Perfect demosaicing is a cheat const uint32_t *pixel = mScene.getPixelElectrons(); rCount = pixel[Scene::R] * scale64x; gCount = pixel[Scene::Gr] * scale64x; bCount = pixel[Scene::B] * scale64x; *px++ = rCount < 255 * 64 ? rCount / 64 : 255; *px++ = gCount < 255 * 64 ? gCount / 64 : 255; *px++ = bCount < 255 * 64 ? bCount / 64 : 255; *px++ = 255; for (unsigned int j = 1; j < inc; j++) mScene.getPixelElectrons(); } // TODO: Handle this better // simulatedTime += mRowReadoutTime; } ALOGVV("RGBA sensor image captured"); } void Sensor::captureRGB(uint8_t *img, uint32_t gain, uint32_t stride) { float totalGain = gain / 100.0 * kBaseGainFactor; // In fixed-point math, calculate total scaling from electrons to 8bpp int scale64x = 64 * totalGain * 255 / kMaxRawValue; uint32_t inc = ceil((float)mResolution[0] / stride); for (unsigned int y = 0, outY = 0; y < mResolution[1]; y += inc, outY++) { mScene.setReadoutPixel(0, y); uint8_t *px = img + outY * stride * 3; for (unsigned int x = 0; x < mResolution[0]; x += inc) { uint32_t rCount, gCount, bCount; // TODO: Perfect demosaicing is a cheat const uint32_t *pixel = mScene.getPixelElectrons(); rCount = pixel[Scene::R] * scale64x; gCount = pixel[Scene::Gr] * scale64x; bCount = pixel[Scene::B] * scale64x; *px++ = rCount < 255 * 64 ? rCount / 64 : 255; *px++ = gCount < 255 * 64 ? gCount / 64 : 255; *px++ = bCount < 255 * 64 ? bCount / 64 : 255; for (unsigned int j = 1; j < inc; j++) mScene.getPixelElectrons(); } // TODO: Handle this better // simulatedTime += mRowReadoutTime; } ALOGVV("RGB sensor image captured"); } void Sensor::captureNV21(uint8_t *img, uint32_t gain, uint32_t stride) { float totalGain = gain / 100.0 * kBaseGainFactor; // Using fixed-point math with 6 bits of fractional precision. // In fixed-point math, calculate total scaling from electrons to 8bpp const int scale64x = 64 * totalGain * 255 / kMaxRawValue; // In fixed-point math, saturation point of sensor after gain const int saturationPoint = 64 * 255; // Fixed-point coefficients for RGB-YUV transform // Based on JFIF RGB->YUV transform. // Cb/Cr offset scaled by 64x twice since they're applied post-multiply const int rgbToY[] = {19, 37, 7}; const int rgbToCb[] = {-10, -21, 32, 524288}; const int rgbToCr[] = {32, -26, -5, 524288}; // Scale back to 8bpp non-fixed-point const int scaleOut = 64; const int scaleOutSq = scaleOut * scaleOut; // after multiplies // inc = how many pixels to skip while reading every next pixel // horizontally. uint32_t inc = ceil((float)mResolution[0] / stride); // outH = projected vertical resolution based on stride. uint32_t outH = mResolution[1] / inc; for (unsigned int y = 0, outY = 0; y < mResolution[1]; y += inc, outY++) { uint8_t *pxY = img + outY * stride; uint8_t *pxVU = img + (outH + outY / 2) * stride; mScene.setReadoutPixel(0, y); for (unsigned int outX = 0; outX < stride; outX++) { int32_t rCount, gCount, bCount; // TODO: Perfect demosaicing is a cheat const uint32_t *pixel = mScene.getPixelElectrons(); rCount = pixel[Scene::R] * scale64x; rCount = rCount < saturationPoint ? rCount : saturationPoint; gCount = pixel[Scene::Gr] * scale64x; gCount = gCount < saturationPoint ? gCount : saturationPoint; bCount = pixel[Scene::B] * scale64x; bCount = bCount < saturationPoint ? bCount : saturationPoint; *pxY++ = (rgbToY[0] * rCount + rgbToY[1] * gCount + rgbToY[2] * bCount) / scaleOutSq; if (outY % 2 == 0 && outX % 2 == 0) { *pxVU++ = (rgbToCb[0] * rCount + rgbToCb[1] * gCount + rgbToCb[2] * bCount + rgbToCb[3]) / scaleOutSq; *pxVU++ = (rgbToCr[0] * rCount + rgbToCr[1] * gCount + rgbToCr[2] * bCount + rgbToCr[3]) / scaleOutSq; } // Skip unprocessed pixels from sensor. for (unsigned int j = 1; j < inc; j++) mScene.getPixelElectrons(); } } ALOGVV("NV21 sensor image captured"); } void Sensor::captureDepth(uint8_t *img, uint32_t gain, uint32_t stride) { float totalGain = gain / 100.0 * kBaseGainFactor; // In fixed-point math, calculate scaling factor to 13bpp millimeters int scale64x = 64 * totalGain * 8191 / kMaxRawValue; uint32_t inc = ceil((float)mResolution[0] / stride); for (unsigned int y = 0, outY = 0; y < mResolution[1]; y += inc, outY++) { mScene.setReadoutPixel(0, y); uint16_t *px = ((uint16_t *)img) + outY * stride; for (unsigned int x = 0; x < mResolution[0]; x += inc) { uint32_t depthCount; // TODO: Make up real depth scene instead of using green channel // as depth const uint32_t *pixel = mScene.getPixelElectrons(); depthCount = pixel[Scene::Gr] * scale64x; *px++ = depthCount < 8191 * 64 ? depthCount / 64 : 0; for (unsigned int j = 1; j < inc; j++) mScene.getPixelElectrons(); } // TODO: Handle this better // simulatedTime += mRowReadoutTime; } ALOGVV("Depth sensor image captured"); } void Sensor::captureDepthCloud(uint8_t * /*img*/) { #if defined HAL_DATASPACE_DEPTH android_depth_points *cloud = reinterpret_cast<android_depth_points *>(img); cloud->num_points = 16; // TODO: Create point cloud values that match RGB scene const int FLOATS_PER_POINT = 4; const float JITTER_STDDEV = 0.1f; for (size_t y = 0, i = 0; y < 4; y++) { for (size_t x = 0; x < 4; x++, i++) { float randSampleX = std::rand() * (2.5f / (1.0f + RAND_MAX)) - 1.25f; randSampleX *= JITTER_STDDEV; float randSampleY = std::rand() * (2.5f / (1.0f + RAND_MAX)) - 1.25f; randSampleY *= JITTER_STDDEV; float randSampleZ = std::rand() * (2.5f / (1.0f + RAND_MAX)) - 1.25f; randSampleZ *= JITTER_STDDEV; cloud->xyzc_points[i * FLOATS_PER_POINT + 0] = x - 1.5f + randSampleX; cloud->xyzc_points[i * FLOATS_PER_POINT + 1] = y - 1.5f + randSampleY; cloud->xyzc_points[i * FLOATS_PER_POINT + 2] = 3.f + randSampleZ; cloud->xyzc_points[i * FLOATS_PER_POINT + 3] = 0.8f; } } ALOGVV("Depth point cloud captured"); #endif } } // namespace android