/*
* Copyright (C) 2019 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
//#define LOG_NDEBUG 0
#define LOG_TAG "C2SoftOpusEnc"
#include <utils/Log.h>
#include <C2PlatformSupport.h>
#include <SimpleC2Interface.h>
#include <media/stagefright/foundation/MediaDefs.h>
#include <media/stagefright/foundation/OpusHeader.h>
#include "C2SoftOpusEnc.h"
extern "C" {
#include <opus.h>
#include <opus_multistream.h>
}
#define DEFAULT_FRAME_DURATION_MS 20
namespace android {
namespace {
constexpr char COMPONENT_NAME[] = "c2.android.opus.encoder";
} // namespace
static const int kMaxNumChannelsSupported = 2;
class C2SoftOpusEnc::IntfImpl : public SimpleInterface<void>::BaseParams {
public:
explicit IntfImpl(const std::shared_ptr<C2ReflectorHelper> &helper)
: SimpleInterface<void>::BaseParams(
helper,
COMPONENT_NAME,
C2Component::KIND_ENCODER,
C2Component::DOMAIN_AUDIO,
MEDIA_MIMETYPE_AUDIO_OPUS) {
noPrivateBuffers();
noInputReferences();
noOutputReferences();
noInputLatency();
noTimeStretch();
setDerivedInstance(this);
addParameter(
DefineParam(mAttrib, C2_PARAMKEY_COMPONENT_ATTRIBUTES)
.withConstValue(new C2ComponentAttributesSetting(
C2Component::ATTRIB_IS_TEMPORAL))
.build());
addParameter(
DefineParam(mSampleRate, C2_PARAMKEY_SAMPLE_RATE)
.withDefault(new C2StreamSampleRateInfo::input(0u, 48000))
.withFields({C2F(mSampleRate, value).oneOf({
8000, 12000, 16000, 24000, 48000})})
.withSetter((Setter<decltype(*mSampleRate)>::StrictValueWithNoDeps))
.build());
addParameter(
DefineParam(mChannelCount, C2_PARAMKEY_CHANNEL_COUNT)
.withDefault(new C2StreamChannelCountInfo::input(0u, 1))
.withFields({C2F(mChannelCount, value).inRange(1, kMaxNumChannelsSupported)})
.withSetter((Setter<decltype(*mChannelCount)>::StrictValueWithNoDeps))
.build());
addParameter(
DefineParam(mBitrate, C2_PARAMKEY_BITRATE)
.withDefault(new C2StreamBitrateInfo::output(0u, 128000))
.withFields({C2F(mBitrate, value).inRange(500, 512000)})
.withSetter(Setter<decltype(*mBitrate)>::NonStrictValueWithNoDeps)
.build());
addParameter(
DefineParam(mComplexity, C2_PARAMKEY_COMPLEXITY)
.withDefault(new C2StreamComplexityTuning::output(0u, 10))
.withFields({C2F(mComplexity, value).inRange(1, 10)})
.withSetter(Setter<decltype(*mComplexity)>::NonStrictValueWithNoDeps)
.build());
addParameter(
DefineParam(mInputMaxBufSize, C2_PARAMKEY_INPUT_MAX_BUFFER_SIZE)
.withConstValue(new C2StreamMaxBufferSizeInfo::input(0u, 3840))
.build());
}
uint32_t getSampleRate() const { return mSampleRate->value; }
uint32_t getChannelCount() const { return mChannelCount->value; }
uint32_t getBitrate() const { return mBitrate->value; }
uint32_t getComplexity() const { return mComplexity->value; }
private:
std::shared_ptr<C2StreamSampleRateInfo::input> mSampleRate;
std::shared_ptr<C2StreamChannelCountInfo::input> mChannelCount;
std::shared_ptr<C2StreamBitrateInfo::output> mBitrate;
std::shared_ptr<C2StreamComplexityTuning::output> mComplexity;
std::shared_ptr<C2StreamMaxBufferSizeInfo::input> mInputMaxBufSize;
};
C2SoftOpusEnc::C2SoftOpusEnc(const char* name, c2_node_id_t id,
const std::shared_ptr<IntfImpl>& intfImpl)
: SimpleC2Component(
std::make_shared<SimpleInterface<IntfImpl>>(name, id, intfImpl)),
mIntf(intfImpl),
mOutputBlock(nullptr),
mEncoder(nullptr),
mInputBufferPcm16(nullptr),
mOutIndex(0u) {
}
C2SoftOpusEnc::~C2SoftOpusEnc() {
onRelease();
}
c2_status_t C2SoftOpusEnc::onInit() {
return initEncoder();
}
c2_status_t C2SoftOpusEnc::configureEncoder() {
static const unsigned char mono_mapping[256] = {0};
static const unsigned char stereo_mapping[256] = {0, 1};
mSampleRate = mIntf->getSampleRate();
mChannelCount = mIntf->getChannelCount();
uint32_t bitrate = mIntf->getBitrate();
int complexity = mIntf->getComplexity();
mNumSamplesPerFrame = mSampleRate / (1000 / mFrameDurationMs);
mNumPcmBytesPerInputFrame =
mChannelCount * mNumSamplesPerFrame * sizeof(int16_t);
int err = C2_OK;
const unsigned char* mapping;
if (mChannelCount == 1) {
mapping = mono_mapping;
} else if (mChannelCount == 2) {
mapping = stereo_mapping;
} else {
ALOGE("Number of channels (%d) is not supported", mChannelCount);
return C2_BAD_VALUE;
}
if (mEncoder != nullptr) {
opus_multistream_encoder_destroy(mEncoder);
}
mEncoder = opus_multistream_encoder_create(mSampleRate, mChannelCount,
1, mChannelCount - 1, mapping, OPUS_APPLICATION_AUDIO, &err);
if (err) {
ALOGE("Could not create libopus encoder. Error code: %i", err);
return C2_CORRUPTED;
}
// Complexity
if (opus_multistream_encoder_ctl(
mEncoder, OPUS_SET_COMPLEXITY(complexity)) != OPUS_OK) {
ALOGE("failed to set complexity");
return C2_BAD_VALUE;
}
// DTX
if (opus_multistream_encoder_ctl(mEncoder, OPUS_SET_DTX(0) != OPUS_OK)) {
ALOGE("failed to set dtx");
return C2_BAD_VALUE;
}
// Application
if (opus_multistream_encoder_ctl(mEncoder,
OPUS_SET_APPLICATION(OPUS_APPLICATION_AUDIO)) != OPUS_OK) {
ALOGE("failed to set application");
return C2_BAD_VALUE;
}
// Signal type
if (opus_multistream_encoder_ctl(mEncoder, OPUS_SET_SIGNAL(OPUS_AUTO)) !=
OPUS_OK) {
ALOGE("failed to set signal");
return C2_BAD_VALUE;
}
// Constrained VBR
if (opus_multistream_encoder_ctl(mEncoder, OPUS_SET_VBR(1) != OPUS_OK)) {
ALOGE("failed to set vbr type");
return C2_BAD_VALUE;
}
if (opus_multistream_encoder_ctl(mEncoder, OPUS_SET_VBR_CONSTRAINT(1) !=
OPUS_OK)) {
ALOGE("failed to set vbr constraint");
return C2_BAD_VALUE;
}
// Bitrate
if (opus_multistream_encoder_ctl(mEncoder, OPUS_SET_BITRATE(bitrate)) !=
OPUS_OK) {
ALOGE("failed to set bitrate");
return C2_BAD_VALUE;
}
// Set seek preroll to 80 ms
mSeekPreRoll = 80000000;
return C2_OK;
}
c2_status_t C2SoftOpusEnc::initEncoder() {
mSignalledEos = false;
mSignalledError = false;
mHeaderGenerated = false;
mIsFirstFrame = true;
mEncoderFlushed = false;
mBufferAvailable = false;
mAnchorTimeStamp = 0ull;
mProcessedSamples = 0;
mFilledLen = 0;
mFrameDurationMs = DEFAULT_FRAME_DURATION_MS;
if (!mInputBufferPcm16) {
mInputBufferPcm16 =
(int16_t*)malloc(kFrameSize * kMaxNumChannels * sizeof(int16_t));
}
if (!mInputBufferPcm16) return C2_NO_MEMORY;
/* Default Configurations */
c2_status_t status = configureEncoder();
return status;
}
c2_status_t C2SoftOpusEnc::onStop() {
mSignalledEos = false;
mSignalledError = false;
mIsFirstFrame = true;
mEncoderFlushed = false;
mBufferAvailable = false;
mAnchorTimeStamp = 0ull;
mProcessedSamples = 0u;
mFilledLen = 0;
if (mEncoder) {
int status = opus_multistream_encoder_ctl(mEncoder, OPUS_RESET_STATE);
if (status != OPUS_OK) {
ALOGE("OPUS_RESET_STATE failed status = %s", opus_strerror(status));
mSignalledError = true;
return C2_CORRUPTED;
}
}
if (mOutputBlock) mOutputBlock.reset();
mOutputBlock = nullptr;
return C2_OK;
}
void C2SoftOpusEnc::onReset() {
(void)onStop();
}
void C2SoftOpusEnc::onRelease() {
(void)onStop();
if (mInputBufferPcm16) {
free(mInputBufferPcm16);
mInputBufferPcm16 = nullptr;
}
if (mEncoder) {
opus_multistream_encoder_destroy(mEncoder);
mEncoder = nullptr;
}
}
c2_status_t C2SoftOpusEnc::onFlush_sm() {
return onStop();
}
// Drain the encoder to get last frames (if any)
int C2SoftOpusEnc::drainEncoder(uint8_t* outPtr) {
memset((uint8_t *)mInputBufferPcm16 + mFilledLen, 0,
(mNumPcmBytesPerInputFrame - mFilledLen));
int encodedBytes = opus_multistream_encode(
mEncoder, mInputBufferPcm16, mNumSamplesPerFrame, outPtr, kMaxPayload);
if (encodedBytes > mOutputBlock->capacity()) {
ALOGE("not enough space left to write encoded data, dropping %d bytes",
mBytesEncoded);
// a fatal error would stop the encoding
return -1;
}
ALOGV("encoded %i Opus bytes from %zu PCM bytes", encodedBytes,
mNumPcmBytesPerInputFrame);
mEncoderFlushed = true;
mFilledLen = 0;
return encodedBytes;
}
void C2SoftOpusEnc::process(const std::unique_ptr<C2Work>& work,
const std::shared_ptr<C2BlockPool>& pool) {
// Initialize output work
work->result = C2_OK;
work->workletsProcessed = 1u;
work->worklets.front()->output.flags = work->input.flags;
if (mSignalledError || mSignalledEos) {
work->result = C2_BAD_VALUE;
return;
}
bool eos = (work->input.flags & C2FrameData::FLAG_END_OF_STREAM) != 0;
C2ReadView rView = mDummyReadView;
size_t inOffset = 0u;
size_t inSize = 0u;
c2_status_t err = C2_OK;
if (!work->input.buffers.empty()) {
rView =
work->input.buffers[0]->data().linearBlocks().front().map().get();
inSize = rView.capacity();
if (inSize && rView.error()) {
ALOGE("read view map failed %d", rView.error());
work->result = C2_CORRUPTED;
return;
}
}
ALOGV("in buffer attr. size %zu timestamp %d frameindex %d, flags %x",
inSize, (int)work->input.ordinal.timestamp.peeku(),
(int)work->input.ordinal.frameIndex.peeku(), work->input.flags);
if (!mEncoder) {
if (initEncoder() != C2_OK) {
ALOGE("initEncoder failed with status %d", err);
work->result = err;
mSignalledError = true;
return;
}
}
if (mIsFirstFrame && inSize > 0) {
mAnchorTimeStamp = work->input.ordinal.timestamp.peekull();
mIsFirstFrame = false;
}
C2MemoryUsage usage = {C2MemoryUsage::CPU_READ, C2MemoryUsage::CPU_WRITE};
err = pool->fetchLinearBlock(kMaxPayload, usage, &mOutputBlock);
if (err != C2_OK) {
ALOGE("fetchLinearBlock for Output failed with status %d", err);
work->result = C2_NO_MEMORY;
return;
}
C2WriteView wView = mOutputBlock->map().get();
if (wView.error()) {
ALOGE("write view map failed %d", wView.error());
work->result = C2_CORRUPTED;
mOutputBlock.reset();
return;
}
size_t inPos = 0;
size_t processSize = 0;
mBytesEncoded = 0;
uint64_t outTimeStamp = 0u;
std::shared_ptr<C2Buffer> buffer;
uint64_t inputIndex = work->input.ordinal.frameIndex.peeku();
const uint8_t* inPtr = rView.data() + inOffset;
class FillWork {
public:
FillWork(uint32_t flags, C2WorkOrdinalStruct ordinal,
const std::shared_ptr<C2Buffer> &buffer)
: mFlags(flags), mOrdinal(ordinal), mBuffer(buffer) {
}
~FillWork() = default;
void operator()(const std::unique_ptr<C2Work>& work) {
work->worklets.front()->output.flags = (C2FrameData::flags_t)mFlags;
work->worklets.front()->output.buffers.clear();
work->worklets.front()->output.ordinal = mOrdinal;
work->workletsProcessed = 1u;
work->result = C2_OK;
if (mBuffer) {
work->worklets.front()->output.buffers.push_back(mBuffer);
}
ALOGV("timestamp = %lld, index = %lld, w/%s buffer",
mOrdinal.timestamp.peekll(),
mOrdinal.frameIndex.peekll(),
mBuffer ? "" : "o");
}
private:
const uint32_t mFlags;
const C2WorkOrdinalStruct mOrdinal;
const std::shared_ptr<C2Buffer> mBuffer;
};
C2WorkOrdinalStruct outOrdinal = work->input.ordinal;
if (!mHeaderGenerated) {
uint8_t header[AOPUS_UNIFIED_CSD_MAXSIZE];
memset(header, 0, sizeof(header));
// Get codecDelay
int32_t lookahead;
if (opus_multistream_encoder_ctl(mEncoder, OPUS_GET_LOOKAHEAD(&lookahead)) !=
OPUS_OK) {
ALOGE("failed to get lookahead");
mSignalledError = true;
work->result = C2_CORRUPTED;
return;
}
mCodecDelay = lookahead * 1000000000ll / mSampleRate;
OpusHeader opusHeader;
memset(&opusHeader, 0, sizeof(opusHeader));
opusHeader.channels = mChannelCount;
opusHeader.num_streams = mChannelCount;
opusHeader.num_coupled = 0;
opusHeader.channel_mapping = ((mChannelCount > 8) ? 255 : (mChannelCount > 2));
opusHeader.gain_db = 0;
opusHeader.skip_samples = lookahead;
int headerLen = WriteOpusHeaders(opusHeader, mSampleRate, header,
sizeof(header), mCodecDelay, mSeekPreRoll);
std::unique_ptr<C2StreamInitDataInfo::output> csd =
C2StreamInitDataInfo::output::AllocUnique(headerLen, 0u);
if (!csd) {
ALOGE("CSD allocation failed");
mSignalledError = true;
work->result = C2_NO_MEMORY;
return;
}
ALOGV("put csd, %d bytes", headerLen);
memcpy(csd->m.value, header, headerLen);
work->worklets.front()->output.configUpdate.push_back(std::move(csd));
mHeaderGenerated = true;
}
/*
* For buffer size which is not a multiple of mNumPcmBytesPerInputFrame, we will
* accumulate the input and keep it. Once the input is filled with expected number
* of bytes, we will send it to encoder. mFilledLen manages the bytes of input yet
* to be processed. The next call will fill mNumPcmBytesPerInputFrame - mFilledLen
* bytes to input and send it to the encoder.
*/
while (inPos < inSize) {
const uint8_t* pcmBytes = inPtr + inPos;
int filledSamples = mFilledLen / sizeof(int16_t);
if ((inPos + (mNumPcmBytesPerInputFrame - mFilledLen)) <= inSize) {
processSize = mNumPcmBytesPerInputFrame - mFilledLen;
mBufferAvailable = true;
} else {
processSize = inSize - inPos;
mBufferAvailable = false;
if (eos) {
memset(mInputBufferPcm16 + filledSamples, 0,
(mNumPcmBytesPerInputFrame - mFilledLen));
mBufferAvailable = true;
}
}
const unsigned nInputSamples = processSize / sizeof(int16_t);
for (unsigned i = 0; i < nInputSamples; i++) {
int32_t data = pcmBytes[2 * i + 1] << 8 | pcmBytes[2 * i];
data = ((data & 0xFFFF) ^ 0x8000) - 0x8000;
mInputBufferPcm16[i + filledSamples] = data;
}
inPos += processSize;
mFilledLen += processSize;
if (!mBufferAvailable) break;
uint8_t* outPtr = wView.data() + mBytesEncoded;
int encodedBytes =
opus_multistream_encode(mEncoder, mInputBufferPcm16,
mNumSamplesPerFrame, outPtr, kMaxPayload - mBytesEncoded);
ALOGV("encoded %i Opus bytes from %zu PCM bytes", encodedBytes,
processSize);
if (encodedBytes < 0 || encodedBytes > (kMaxPayload - mBytesEncoded)) {
ALOGE("opus_encode failed, encodedBytes : %d", encodedBytes);
mSignalledError = true;
work->result = C2_CORRUPTED;
return;
}
if (buffer) {
outOrdinal.frameIndex = mOutIndex++;
outOrdinal.timestamp = mAnchorTimeStamp + outTimeStamp;
cloneAndSend(
inputIndex, work,
FillWork(C2FrameData::FLAG_INCOMPLETE, outOrdinal, buffer));
buffer.reset();
}
if (encodedBytes > 0) {
buffer =
createLinearBuffer(mOutputBlock, mBytesEncoded, encodedBytes);
}
mBytesEncoded += encodedBytes;
mProcessedSamples += (filledSamples + nInputSamples);
outTimeStamp =
mProcessedSamples * 1000000ll / mChannelCount / mSampleRate;
if ((processSize + mFilledLen) < mNumPcmBytesPerInputFrame)
mEncoderFlushed = true;
mFilledLen = 0;
}
uint32_t flags = 0;
if (eos) {
ALOGV("signalled eos");
mSignalledEos = true;
if (!mEncoderFlushed) {
if (buffer) {
outOrdinal.frameIndex = mOutIndex++;
outOrdinal.timestamp = mAnchorTimeStamp + outTimeStamp;
cloneAndSend(
inputIndex, work,
FillWork(C2FrameData::FLAG_INCOMPLETE, outOrdinal, buffer));
buffer.reset();
}
// drain the encoder for last buffer
drainInternal(pool, work);
}
flags = C2FrameData::FLAG_END_OF_STREAM;
}
if (buffer) {
outOrdinal.frameIndex = mOutIndex++;
outOrdinal.timestamp = mAnchorTimeStamp + outTimeStamp;
FillWork((C2FrameData::flags_t)(flags), outOrdinal, buffer)(work);
buffer.reset();
}
mOutputBlock = nullptr;
}
c2_status_t C2SoftOpusEnc::drainInternal(
const std::shared_ptr<C2BlockPool>& pool,
const std::unique_ptr<C2Work>& work) {
mBytesEncoded = 0;
std::shared_ptr<C2Buffer> buffer = nullptr;
C2WorkOrdinalStruct outOrdinal = work->input.ordinal;
bool eos = (work->input.flags & C2FrameData::FLAG_END_OF_STREAM) != 0;
C2MemoryUsage usage = {C2MemoryUsage::CPU_READ, C2MemoryUsage::CPU_WRITE};
c2_status_t err = pool->fetchLinearBlock(kMaxPayload, usage, &mOutputBlock);
if (err != C2_OK) {
ALOGE("fetchLinearBlock for Output failed with status %d", err);
return C2_NO_MEMORY;
}
C2WriteView wView = mOutputBlock->map().get();
if (wView.error()) {
ALOGE("write view map failed %d", wView.error());
mOutputBlock.reset();
return C2_CORRUPTED;
}
int encBytes = drainEncoder(wView.data());
if (encBytes > 0) mBytesEncoded += encBytes;
if (mBytesEncoded > 0) {
buffer = createLinearBuffer(mOutputBlock, 0, mBytesEncoded);
mOutputBlock.reset();
}
mProcessedSamples += (mNumPcmBytesPerInputFrame / sizeof(int16_t));
uint64_t outTimeStamp =
mProcessedSamples * 1000000ll / mChannelCount / mSampleRate;
outOrdinal.frameIndex = mOutIndex++;
outOrdinal.timestamp = mAnchorTimeStamp + outTimeStamp;
work->worklets.front()->output.flags =
(C2FrameData::flags_t)(eos ? C2FrameData::FLAG_END_OF_STREAM : 0);
work->worklets.front()->output.buffers.clear();
work->worklets.front()->output.ordinal = outOrdinal;
work->workletsProcessed = 1u;
work->result = C2_OK;
if (buffer) {
work->worklets.front()->output.buffers.push_back(buffer);
}
mOutputBlock = nullptr;
return C2_OK;
}
c2_status_t C2SoftOpusEnc::drain(uint32_t drainMode,
const std::shared_ptr<C2BlockPool>& pool) {
if (drainMode == NO_DRAIN) {
ALOGW("drain with NO_DRAIN: no-op");
return C2_OK;
}
if (drainMode == DRAIN_CHAIN) {
ALOGW("DRAIN_CHAIN not supported");
return C2_OMITTED;
}
mIsFirstFrame = true;
mAnchorTimeStamp = 0ull;
mProcessedSamples = 0u;
return drainInternal(pool, nullptr);
}
class C2SoftOpusEncFactory : public C2ComponentFactory {
public:
C2SoftOpusEncFactory()
: mHelper(std::static_pointer_cast<C2ReflectorHelper>(
GetCodec2PlatformComponentStore()->getParamReflector())) {}
virtual c2_status_t createComponent(
c2_node_id_t id, std::shared_ptr<C2Component>* const component,
std::function<void(C2Component*)> deleter) override {
*component = std::shared_ptr<C2Component>(
new C2SoftOpusEnc(
COMPONENT_NAME, id,
std::make_shared<C2SoftOpusEnc::IntfImpl>(mHelper)),
deleter);
return C2_OK;
}
virtual c2_status_t createInterface(
c2_node_id_t id, std::shared_ptr<C2ComponentInterface>* const interface,
std::function<void(C2ComponentInterface*)> deleter) override {
*interface = std::shared_ptr<C2ComponentInterface>(
new SimpleInterface<C2SoftOpusEnc::IntfImpl>(
COMPONENT_NAME, id,
std::make_shared<C2SoftOpusEnc::IntfImpl>(mHelper)),
deleter);
return C2_OK;
}
virtual ~C2SoftOpusEncFactory() override = default;
private:
std::shared_ptr<C2ReflectorHelper> mHelper;
};
} // namespace android
extern "C" ::C2ComponentFactory* CreateCodec2Factory() {
ALOGV("in %s", __func__);
return new ::android::C2SoftOpusEncFactory();
}
extern "C" void DestroyCodec2Factory(::C2ComponentFactory* factory) {
ALOGV("in %s", __func__);
delete factory;
}