/* * Copyright 2012, The Android Open Source Project * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ //#define LOG_NDEBUG 0 #define LOG_TAG "Converter" #include #include "Converter.h" #include "MediaPuller.h" #include #include #include #include #include #include #include #include #include #include namespace android { Converter::Converter( const sp ¬ify, const sp &codecLooper, const sp &format, bool usePCMAudio) : mInitCheck(NO_INIT), mNotify(notify), mCodecLooper(codecLooper), mInputFormat(format), mIsVideo(false), mIsPCMAudio(usePCMAudio), mDoMoreWorkPending(false) #if ENABLE_SILENCE_DETECTION ,mFirstSilentFrameUs(-1ll) ,mInSilentMode(false) #endif { AString mime; CHECK(mInputFormat->findString("mime", &mime)); if (!strncasecmp("video/", mime.c_str(), 6)) { mIsVideo = true; } CHECK(!usePCMAudio || !mIsVideo); mInitCheck = initEncoder(); if (mInitCheck != OK) { if (mEncoder != NULL) { mEncoder->release(); mEncoder.clear(); } } } Converter::~Converter() { CHECK(mEncoder == NULL); } void Converter::shutdownAsync() { ALOGV("shutdown"); (new AMessage(kWhatShutdown, id()))->post(); } status_t Converter::initCheck() const { return mInitCheck; } size_t Converter::getInputBufferCount() const { return mEncoderInputBuffers.size(); } sp Converter::getOutputFormat() const { return mOutputFormat; } static int32_t getBitrate(const char *propName, int32_t defaultValue) { char val[PROPERTY_VALUE_MAX]; if (property_get(propName, val, NULL)) { char *end; unsigned long x = strtoul(val, &end, 10); if (*end == '\0' && end > val && x > 0) { return x; } } return defaultValue; } status_t Converter::initEncoder() { AString inputMIME; CHECK(mInputFormat->findString("mime", &inputMIME)); AString outputMIME; bool isAudio = false; if (!strcasecmp(inputMIME.c_str(), MEDIA_MIMETYPE_AUDIO_RAW)) { if (mIsPCMAudio) { outputMIME = MEDIA_MIMETYPE_AUDIO_RAW; } else { outputMIME = MEDIA_MIMETYPE_AUDIO_AAC; } isAudio = true; } else if (!strcasecmp(inputMIME.c_str(), MEDIA_MIMETYPE_VIDEO_RAW)) { outputMIME = MEDIA_MIMETYPE_VIDEO_AVC; } else { TRESPASS(); } if (!mIsPCMAudio) { mEncoder = MediaCodec::CreateByType( mCodecLooper, outputMIME.c_str(), true /* encoder */); if (mEncoder == NULL) { return ERROR_UNSUPPORTED; } } mOutputFormat = mInputFormat->dup(); if (mIsPCMAudio) { return OK; } mOutputFormat->setString("mime", outputMIME.c_str()); int32_t audioBitrate = getBitrate("media.wfd.audio-bitrate", 128000); int32_t videoBitrate = getBitrate("media.wfd.video-bitrate", 5000000); ALOGI("using audio bitrate of %d bps, video bitrate of %d bps", audioBitrate, videoBitrate); if (isAudio) { mOutputFormat->setInt32("bitrate", audioBitrate); } else { mOutputFormat->setInt32("bitrate", videoBitrate); mOutputFormat->setInt32("frame-rate", 30); mOutputFormat->setInt32("i-frame-interval", 1); // Iframes every 1 secs mOutputFormat->setInt32("prepend-sps-pps-to-idr-frames", 1); } ALOGV("output format is '%s'", mOutputFormat->debugString(0).c_str()); status_t err = mEncoder->configure( mOutputFormat, NULL /* nativeWindow */, NULL /* crypto */, MediaCodec::CONFIGURE_FLAG_ENCODE); if (err != OK) { return err; } err = mEncoder->start(); if (err != OK) { return err; } err = mEncoder->getInputBuffers(&mEncoderInputBuffers); if (err != OK) { return err; } return mEncoder->getOutputBuffers(&mEncoderOutputBuffers); } void Converter::notifyError(status_t err) { sp notify = mNotify->dup(); notify->setInt32("what", kWhatError); notify->setInt32("err", err); notify->post(); } // static bool Converter::IsSilence(const sp &accessUnit) { const uint8_t *ptr = accessUnit->data(); const uint8_t *end = ptr + accessUnit->size(); while (ptr < end) { if (*ptr != 0) { return false; } ++ptr; } return true; } void Converter::onMessageReceived(const sp &msg) { switch (msg->what()) { case kWhatMediaPullerNotify: { int32_t what; CHECK(msg->findInt32("what", &what)); if (!mIsPCMAudio && mEncoder == NULL) { ALOGV("got msg '%s' after encoder shutdown.", msg->debugString().c_str()); if (what == MediaPuller::kWhatAccessUnit) { sp accessUnit; CHECK(msg->findBuffer("accessUnit", &accessUnit)); void *mbuf; if (accessUnit->meta()->findPointer("mediaBuffer", &mbuf) && mbuf != NULL) { ALOGV("releasing mbuf %p", mbuf); accessUnit->meta()->setPointer("mediaBuffer", NULL); static_cast(mbuf)->release(); mbuf = NULL; } } break; } if (what == MediaPuller::kWhatEOS) { mInputBufferQueue.push_back(NULL); feedEncoderInputBuffers(); scheduleDoMoreWork(); } else { CHECK_EQ(what, MediaPuller::kWhatAccessUnit); sp accessUnit; CHECK(msg->findBuffer("accessUnit", &accessUnit)); #if 0 void *mbuf; if (accessUnit->meta()->findPointer("mediaBuffer", &mbuf) && mbuf != NULL) { ALOGI("queueing mbuf %p", mbuf); } #endif #if ENABLE_SILENCE_DETECTION if (!mIsVideo) { if (IsSilence(accessUnit)) { if (mInSilentMode) { break; } int64_t nowUs = ALooper::GetNowUs(); if (mFirstSilentFrameUs < 0ll) { mFirstSilentFrameUs = nowUs; } else if (nowUs >= mFirstSilentFrameUs + 10000000ll) { mInSilentMode = true; ALOGI("audio in silent mode now."); break; } } else { if (mInSilentMode) { ALOGI("audio no longer in silent mode."); } mInSilentMode = false; mFirstSilentFrameUs = -1ll; } } #endif mInputBufferQueue.push_back(accessUnit); feedEncoderInputBuffers(); scheduleDoMoreWork(); } break; } case kWhatEncoderActivity: { #if 0 int64_t whenUs; if (msg->findInt64("whenUs", &whenUs)) { int64_t nowUs = ALooper::GetNowUs(); ALOGI("[%s] kWhatEncoderActivity after %lld us", mIsVideo ? "video" : "audio", nowUs - whenUs); } #endif mDoMoreWorkPending = false; if (mEncoder == NULL) { break; } status_t err = doMoreWork(); if (err != OK) { notifyError(err); } else { scheduleDoMoreWork(); } break; } case kWhatRequestIDRFrame: { if (mEncoder == NULL) { break; } if (mIsVideo) { ALOGI("requesting IDR frame"); mEncoder->requestIDRFrame(); } break; } case kWhatShutdown: { ALOGI("shutting down encoder"); if (mEncoder != NULL) { mEncoder->release(); mEncoder.clear(); } AString mime; CHECK(mInputFormat->findString("mime", &mime)); ALOGI("encoder (%s) shut down.", mime.c_str()); break; } default: TRESPASS(); } } void Converter::scheduleDoMoreWork() { if (mIsPCMAudio) { // There's no encoder involved in this case. return; } if (mDoMoreWorkPending) { return; } mDoMoreWorkPending = true; #if 1 if (mEncoderActivityNotify == NULL) { mEncoderActivityNotify = new AMessage(kWhatEncoderActivity, id()); } mEncoder->requestActivityNotification(mEncoderActivityNotify->dup()); #else sp notify = new AMessage(kWhatEncoderActivity, id()); notify->setInt64("whenUs", ALooper::GetNowUs()); mEncoder->requestActivityNotification(notify); #endif } status_t Converter::feedRawAudioInputBuffers() { // Split incoming PCM audio into buffers of 6 AUs of 80 audio frames each // and add a 4 byte header according to the wifi display specs. while (!mInputBufferQueue.empty()) { sp buffer = *mInputBufferQueue.begin(); mInputBufferQueue.erase(mInputBufferQueue.begin()); int16_t *ptr = (int16_t *)buffer->data(); int16_t *stop = (int16_t *)(buffer->data() + buffer->size()); while (ptr < stop) { *ptr = htons(*ptr); ++ptr; } static const size_t kFrameSize = 2 * sizeof(int16_t); // stereo static const size_t kFramesPerAU = 80; static const size_t kNumAUsPerPESPacket = 6; if (mPartialAudioAU != NULL) { size_t bytesMissingForFullAU = kNumAUsPerPESPacket * kFramesPerAU * kFrameSize - mPartialAudioAU->size() + 4; size_t copy = buffer->size(); if(copy > bytesMissingForFullAU) { copy = bytesMissingForFullAU; } memcpy(mPartialAudioAU->data() + mPartialAudioAU->size(), buffer->data(), copy); mPartialAudioAU->setRange(0, mPartialAudioAU->size() + copy); buffer->setRange(buffer->offset() + copy, buffer->size() - copy); int64_t timeUs; CHECK(buffer->meta()->findInt64("timeUs", &timeUs)); int64_t copyUs = (int64_t)((copy / kFrameSize) * 1E6 / 48000.0); timeUs += copyUs; buffer->meta()->setInt64("timeUs", timeUs); if (bytesMissingForFullAU == copy) { sp notify = mNotify->dup(); notify->setInt32("what", kWhatAccessUnit); notify->setBuffer("accessUnit", mPartialAudioAU); notify->post(); mPartialAudioAU.clear(); } } while (buffer->size() > 0) { sp partialAudioAU = new ABuffer( 4 + kNumAUsPerPESPacket * kFrameSize * kFramesPerAU); uint8_t *ptr = partialAudioAU->data(); ptr[0] = 0xa0; // 10100000b ptr[1] = kNumAUsPerPESPacket; ptr[2] = 0; // reserved, audio _emphasis_flag = 0 static const unsigned kQuantizationWordLength = 0; // 16-bit static const unsigned kAudioSamplingFrequency = 2; // 48Khz static const unsigned kNumberOfAudioChannels = 1; // stereo ptr[3] = (kQuantizationWordLength << 6) | (kAudioSamplingFrequency << 3) | kNumberOfAudioChannels; size_t copy = buffer->size(); if (copy > partialAudioAU->size() - 4) { copy = partialAudioAU->size() - 4; } memcpy(&ptr[4], buffer->data(), copy); partialAudioAU->setRange(0, 4 + copy); buffer->setRange(buffer->offset() + copy, buffer->size() - copy); int64_t timeUs; CHECK(buffer->meta()->findInt64("timeUs", &timeUs)); partialAudioAU->meta()->setInt64("timeUs", timeUs); int64_t copyUs = (int64_t)((copy / kFrameSize) * 1E6 / 48000.0); timeUs += copyUs; buffer->meta()->setInt64("timeUs", timeUs); if (copy == partialAudioAU->size() - 4) { sp notify = mNotify->dup(); notify->setInt32("what", kWhatAccessUnit); notify->setBuffer("accessUnit", partialAudioAU); notify->post(); partialAudioAU.clear(); continue; } mPartialAudioAU = partialAudioAU; } } return OK; } status_t Converter::feedEncoderInputBuffers() { if (mIsPCMAudio) { return feedRawAudioInputBuffers(); } while (!mInputBufferQueue.empty() && !mAvailEncoderInputIndices.empty()) { sp buffer = *mInputBufferQueue.begin(); mInputBufferQueue.erase(mInputBufferQueue.begin()); size_t bufferIndex = *mAvailEncoderInputIndices.begin(); mAvailEncoderInputIndices.erase(mAvailEncoderInputIndices.begin()); int64_t timeUs = 0ll; uint32_t flags = 0; if (buffer != NULL) { CHECK(buffer->meta()->findInt64("timeUs", &timeUs)); memcpy(mEncoderInputBuffers.itemAt(bufferIndex)->data(), buffer->data(), buffer->size()); void *mediaBuffer; if (buffer->meta()->findPointer("mediaBuffer", &mediaBuffer) && mediaBuffer != NULL) { mEncoderInputBuffers.itemAt(bufferIndex)->meta() ->setPointer("mediaBuffer", mediaBuffer); buffer->meta()->setPointer("mediaBuffer", NULL); } } else { flags = MediaCodec::BUFFER_FLAG_EOS; } status_t err = mEncoder->queueInputBuffer( bufferIndex, 0, (buffer == NULL) ? 0 : buffer->size(), timeUs, flags); if (err != OK) { return err; } } return OK; } status_t Converter::doMoreWork() { status_t err; for (;;) { size_t bufferIndex; err = mEncoder->dequeueInputBuffer(&bufferIndex); if (err != OK) { break; } mAvailEncoderInputIndices.push_back(bufferIndex); } feedEncoderInputBuffers(); for (;;) { size_t bufferIndex; size_t offset; size_t size; int64_t timeUs; uint32_t flags; err = mEncoder->dequeueOutputBuffer( &bufferIndex, &offset, &size, &timeUs, &flags); if (err != OK) { if (err == -EAGAIN) { err = OK; } break; } if (flags & MediaCodec::BUFFER_FLAG_EOS) { sp notify = mNotify->dup(); notify->setInt32("what", kWhatEOS); notify->post(); } else { sp buffer = new ABuffer(size); buffer->meta()->setInt64("timeUs", timeUs); ALOGV("[%s] time %lld us (%.2f secs)", mIsVideo ? "video" : "audio", timeUs, timeUs / 1E6); memcpy(buffer->data(), mEncoderOutputBuffers.itemAt(bufferIndex)->base() + offset, size); if (flags & MediaCodec::BUFFER_FLAG_CODECCONFIG) { mOutputFormat->setBuffer("csd-0", buffer); } else { sp notify = mNotify->dup(); notify->setInt32("what", kWhatAccessUnit); notify->setBuffer("accessUnit", buffer); notify->post(); } } mEncoder->releaseOutputBuffer(bufferIndex); if (flags & MediaCodec::BUFFER_FLAG_EOS) { break; } } return err; } void Converter::requestIDRFrame() { (new AMessage(kWhatRequestIDRFrame, id()))->post(); } } // namespace android