TimedTextSRTSource::TimedTextSRTSource(const sp<DataSource>& dataSource)
: mSource(dataSource),
mMetaData(new MetaData),
mIndex(0) {
#ifdef MTK_SUBTITLE_SUPPORT
mFileEncodeType = ENCODE_TYPE_NORMAL;
#endif
#ifdef SELF_TEST
scanFile();
int64_t startTimeUs = 0;
int64_t endTimeUs = 0;
Parcel parcel;
MediaSource::ReadOptions options;
status_t err =OK;
int len =7;
int st[] = {888, 18111, 22000, 28888, 38000, 54800, 76000}; //ms
for(int i =0; i< len; i++){
int64_t temp = st[i] * 1000ll; //us
options.setSeekTo(temp);
err = read(&startTimeUs, &endTimeUs, &parcel, &options);
ALOGE("[--SELF_TEST--] seekTime=%lld, getStartTime=%lld, getEndTime=%lld, isReadSuccessfully:%d", temp, startTimeUs, endTimeUs, err);
}
#endif
// TODO: Need to detect the language, because SRT doesn't give language
// information explicitly.
mMetaData->setCString(kKeyMediaLanguage, "und");
}
TimedTextSUBSource::TimedTextSUBSource(const sp<DataSource>& dataSource)
: mSource(dataSource),
mMetaData(new MetaData),
mIndex(0),
mFrameRate(TimedTextUtil::DEFAULT_FRAME_RATE){
mFileEncodeType = ENCODE_TYPE_NORMAL;
#ifdef SELF_TEST
scanFile();
int64_t startTimeUs = 0;
int64_t endTimeUs = 0;
Parcel parcel;
MediaSource::ReadOptions options;
status_t err =OK;
int len =7;
int st[] = {888, 18111, 22000, 28888, 38000, 54800, 76000}; //ms
for(int i =0; i< len; i++){
int64_t temp = st[i] * 1000ll; //us
options.setSeekTo(temp);
err = read(&startTimeUs, &endTimeUs, &parcel, &options);
ALOGE("[--SELF_TEST--] seekTime=%lld, getStartTime=%lld, getEndTime=%lld, isReadSuccessfully:%d", temp, startTimeUs, endTimeUs, err);
}
#endif
}
//==============================External SP Case===============================
TimedTextASSSource::TimedTextASSSource(const sp<DataSource>& dataSource)
: mExSource(dataSource),
mExMetaData(new MetaData),
mExIndex(0) {
mASSFlag = TextDescriptions::OUT_OF_BAND_TEXT_ASS;
#ifdef SELF_TEST
scanFile();
int64_t startTimeUs = 0;
int64_t endTimeUs = 0;
Parcel parcel;
MediaSource::ReadOptions options;
status_t err =OK;
int len =7;
int st[] = {888, 18111, 22000, 28888, 38000, 54800, 76000}; //ms
for(int i =0; i< len; i++){
int64_t temp = st[i] * 1000ll; //us
options.setSeekTo(temp);
err = read(&startTimeUs, &endTimeUs, &parcel, &options);
ALOGE("[--SELF_TEST--] seekTime=%lld, getStartTime=%lld, getEndTime=%lld, isReadSuccessfully:%d", temp, startTimeUs, endTimeUs, err);
}
#endif
}
bool
OmxDecoder::ReadAudio(AudioFrame *aFrame, int64_t aSeekTimeUs)
{
status_t err;
if (mAudioMetadataRead && aSeekTimeUs == -1) {
// Use the data read into the buffer during metadata time
err = OK;
}
else {
ReleaseAudioBuffer();
if (aSeekTimeUs != -1) {
MediaSource::ReadOptions options;
options.setSeekTo(aSeekTimeUs);
err = mAudioSource->read(&mAudioBuffer, &options);
} else {
err = mAudioSource->read(&mAudioBuffer);
}
}
mAudioMetadataRead = false;
aSeekTimeUs = -1;
aFrame->mSize = 0;
if (err == OK && mAudioBuffer && mAudioBuffer->range_length() != 0) {
int64_t timeUs;
if (!mAudioBuffer->meta_data()->findInt64(kKeyTime, &timeUs))
return false;
return ToAudioFrame(aFrame, timeUs,
mAudioBuffer->data(),
mAudioBuffer->range_offset(),
mAudioBuffer->range_length(),
mAudioChannels, mAudioSampleRate);
}
else if (err == INFO_FORMAT_CHANGED) {
// If the format changed, update our cached info.
if (!SetAudioFormat()) {
return false;
} else {
return ReadAudio(aFrame, aSeekTimeUs);
}
}
else if (err == ERROR_END_OF_STREAM) {
if (aFrame->mSize == 0) {
return false;
}
}
else if (err == -ETIMEDOUT) {
LOG(LogLevel::Debug, "OmxDecoder::ReadAudio timed out, will retry");
return true;
}
else if (err != OK) {
LOG(LogLevel::Debug, "OmxDecoder::ReadAudio failed, err=%d", err);
return false;
}
return true;
}
TEST_F(TimedTextSRTSourceTest, seekTimeIsEarlierThanFirst) {
MediaSource::ReadOptions options;
options.setSeekTo(500, MediaSource::ReadOptions::SEEK_PREVIOUS_SYNC);
err = mSource->read(&startTimeUs, &endTimeUs, &parcel, &options);
EXPECT_EQ(OK, err);
EXPECT_EQ(1 * kSecToUsec, startTimeUs);
CheckStartTimeMs(parcel, 1 * kSecToMsec);
}
bool OmxDecoder::ReadAudio(AudioFrame *aFrame, int64_t aSeekTimeUs)
{
MOZ_ASSERT(aSeekTimeUs >= -1);
status_t err;
if (mAudioMetadataRead && aSeekTimeUs == -1) {
// Use the data read into the buffer during metadata time
err = OK;
}
else {
ReleaseAudioBuffer();
if (aSeekTimeUs != -1) {
MediaSource::ReadOptions options;
options.setSeekTo(aSeekTimeUs);
err = mAudioSource->read(&mAudioBuffer, &options);
} else {
err = mAudioSource->read(&mAudioBuffer);
}
}
mAudioMetadataRead = false;
aSeekTimeUs = -1;
if (err == OK && mAudioBuffer->range_length() != 0) {
int64_t timeUs;
if (!mAudioBuffer->meta_data()->findInt64(kKeyTime, &timeUs)) {
LOG("no frame time");
return false;
}
if (timeUs < 0) {
LOG("frame time %lld must be nonnegative", timeUs);
return false;
}
return ToAudioFrame(aFrame, timeUs,
mAudioBuffer->data(),
mAudioBuffer->range_offset(),
mAudioBuffer->range_length(),
mAudioChannels, mAudioSampleRate);
}
else if (err == INFO_FORMAT_CHANGED) {
// If the format changed, update our cached info.
LOG("mAudioSource INFO_FORMAT_CHANGED");
if (!SetAudioFormat())
return false;
else
return ReadAudio(aFrame, aSeekTimeUs);
}
else if (err == ERROR_END_OF_STREAM) {
LOG("mAudioSource END_OF_STREAM");
}
else if (err != OK) {
LOG("mAudioSource ERROR %#x", err);
}
return err == OK;
}
TEST_F(TimedTextSRTSourceTest, seekTimeIsLaterThanLast) {
MediaSource::ReadOptions options;
options.setSeekTo(7 * kSecToUsec, MediaSource::ReadOptions::SEEK_PREVIOUS_SYNC);
err = mSource->read(&startTimeUs, &endTimeUs, &parcel, &options);
EXPECT_EQ(ERROR_END_OF_STREAM, err);
options.setSeekTo(8 * kSecToUsec, MediaSource::ReadOptions::SEEK_PREVIOUS_SYNC);
err = mSource->read(&startTimeUs, &endTimeUs, &parcel, &options);
EXPECT_EQ(ERROR_END_OF_STREAM, err);
}
static void performSeekTest(const sp<MediaSource> &source) {
CHECK_EQ((status_t)OK, source->start());
int64_t durationUs;
CHECK(source->getFormat()->findInt64(kKeyDuration, &durationUs));
for (int64_t seekTimeUs = 0; seekTimeUs <= durationUs;
seekTimeUs += 60000ll) {
MediaSource::ReadOptions options;
options.setSeekTo(
seekTimeUs, MediaSource::ReadOptions::SEEK_PREVIOUS_SYNC);
MediaBuffer *buffer;
status_t err;
for (;;) {
err = source->read(&buffer, &options);
options.clearSeekTo();
if (err == INFO_FORMAT_CHANGED) {
CHECK(buffer == NULL);
continue;
}
if (err != OK) {
CHECK(buffer == NULL);
break;
}
if (buffer->range_length() > 0) {
break;
}
CHECK(buffer != NULL);
buffer->release();
buffer = NULL;
}
if (err == OK) {
int64_t timeUs;
CHECK(buffer->meta_data()->findInt64(kKeyTime, &timeUs));
printf("%lld\t%lld\t%lld\n", seekTimeUs, timeUs, seekTimeUs - timeUs);
buffer->release();
buffer = NULL;
} else {
printf("ERROR\n");
break;
}
}
CHECK_EQ((status_t)OK, source->stop());
}
ssize_t NuMediaExtractor::fetchTrackSamples(
int64_t seekTimeUs, MediaSource::ReadOptions::SeekMode mode) {
TrackInfo *minInfo = NULL;
ssize_t minIndex = -1;
for (size_t i = 0; i < mSelectedTracks.size(); ++i) {
TrackInfo *info = &mSelectedTracks.editItemAt(i);
if (seekTimeUs >= 0ll) {
info->mFinalResult = OK;
if (info->mSample != NULL) {
info->mSample->release();
info->mSample = NULL;
info->mSampleTimeUs = -1ll;
}
} else if (info->mFinalResult != OK) {
continue;
}
if (info->mSample == NULL) {
MediaSource::ReadOptions options;
if (seekTimeUs >= 0ll) {
options.setSeekTo(seekTimeUs, mode);
}
status_t err = info->mSource->read(&info->mSample, &options);
if (err != OK) {
CHECK(info->mSample == NULL);
info->mFinalResult = err;
if (info->mFinalResult != ERROR_END_OF_STREAM) {
ALOGW("read on track %d failed with error %d",
info->mTrackIndex, err);
}
info->mSampleTimeUs = -1ll;
continue;
} else {
CHECK(info->mSample != NULL);
CHECK(info->mSample->meta_data()->findInt64(
kKeyTime, &info->mSampleTimeUs));
}
}
if (minInfo == NULL || info->mSampleTimeUs < minInfo->mSampleTimeUs) {
minInfo = info;
minIndex = i;
}
}
return minIndex;
}
TEST_F(TimedTextSRTSourceTest, seekTimeInBetweenTwo) {
for (int i = 1; i <= 4; i++) {
MediaSource::ReadOptions options;
options.setSeekTo(i * kSecToUsec + 500000, MediaSource::ReadOptions::SEEK_PREVIOUS_SYNC);
err = mSource->read(&startTimeUs, &endTimeUs, &parcel, &options);
EXPECT_EQ(OK, err);
EXPECT_EQ((i + 1) * kSecToUsec, startTimeUs);
options.setSeekTo(i * kSecToUsec + 600000, MediaSource::ReadOptions::SEEK_PREVIOUS_SYNC);
err = mSource->read(&startTimeUs, &endTimeUs, &parcel, &options);
EXPECT_EQ(OK, err);
EXPECT_EQ((i + 1) * kSecToUsec, startTimeUs);
}
}
TEST_F(TimedTextSRTSourceTest, seekTimeIsMatched) {
for (int i = 1; i <= 5; i++) {
MediaSource::ReadOptions options;
options.setSeekTo(i * kSecToUsec, MediaSource::ReadOptions::SEEK_PREVIOUS_SYNC);
err = mSource->read(&startTimeUs, &endTimeUs, &parcel, &options);
EXPECT_EQ(OK, err);
EXPECT_EQ(i * kSecToUsec, startTimeUs);
options.setSeekTo(i * kSecToUsec + 100, MediaSource::ReadOptions::SEEK_PREVIOUS_SYNC);
err = mSource->read(&startTimeUs, &endTimeUs, &parcel, &options);
EXPECT_EQ(OK, err);
EXPECT_EQ(i * kSecToUsec, startTimeUs);
}
}
bool OmxDecoder::ReadAudio(AudioFrame *aFrame, int64_t aSeekTimeUs)
{
status_t err;
if (mAudioMetadataRead && aSeekTimeUs == -1) {
// Use the data read into the buffer during metadata time
err = OK;
}
else {
ReleaseAudioBuffer();
if (aSeekTimeUs != -1) {
MediaSource::ReadOptions options;
options.setSeekTo(aSeekTimeUs);
err = mAudioSource->read(&mAudioBuffer, &options);
} else {
err = mAudioSource->read(&mAudioBuffer);
}
}
mAudioMetadataRead = false;
aSeekTimeUs = -1;
if (err == OK && mAudioBuffer->range_length() != 0) {
int64_t timeUs;
if (!mAudioBuffer->meta_data()->findInt64(kKeyTime, &timeUs))
return false;
return ToAudioFrame(aFrame, timeUs,
mAudioBuffer->data(),
mAudioBuffer->range_offset(),
mAudioBuffer->range_length(),
mAudioChannels, mAudioSampleRate);
}
else if (err == INFO_FORMAT_CHANGED && !SetAudioFormat()) {
// If the format changed, update our cached info.
return false;
}
else if (err == ERROR_END_OF_STREAM)
return false;
return true;
}
TEST_F(TimedTextSRTSourceTest, checkEdgeCase) {
MediaSource::ReadOptions options;
options.setSeekTo(5500 * kMsecToUsec, MediaSource::ReadOptions::SEEK_PREVIOUS_SYNC);
err = mSource->read(&startTimeUs, &endTimeUs, &parcel, &options);
EXPECT_EQ(OK, err);
EXPECT_EQ(5500 * kMsecToUsec, startTimeUs);
subtitle = AStringPrintf("6\n\n");
CheckDataEquals(parcel, subtitle.c_str());
options.setSeekTo(5800 * kMsecToUsec, MediaSource::ReadOptions::SEEK_PREVIOUS_SYNC);
err = mSource->read(&startTimeUs, &endTimeUs, &parcel, &options);
EXPECT_EQ(OK, err);
EXPECT_EQ(5800 * kMsecToUsec, startTimeUs);
subtitle = AStringPrintf("7\n\n");
CheckDataEquals(parcel, subtitle.c_str());
options.setSeekTo(6000 * kMsecToUsec, MediaSource::ReadOptions::SEEK_PREVIOUS_SYNC);
err = mSource->read(&startTimeUs, &endTimeUs, &parcel, &options);
EXPECT_EQ(OK, err);
EXPECT_EQ(6000 * kMsecToUsec, startTimeUs);
subtitle = AStringPrintf("8\n\n");
CheckDataEquals(parcel, subtitle.c_str());
}
bool
OmxDecoder::ReadVideo(VideoFrame *aFrame, int64_t aTimeUs,
bool aKeyframeSkip, bool aDoSeek)
{
if (!mVideoSource.get())
return false;
ReleaseVideoBuffer();
status_t err;
if (aDoSeek) {
{
Mutex::Autolock autoLock(mSeekLock);
ReleaseAllPendingVideoBuffersLocked();
mIsVideoSeeking = true;
}
MediaSource::ReadOptions options;
MediaSource::ReadOptions::SeekMode seekMode;
// If the last timestamp of decoded frame is smaller than seekTime,
// seek to next key frame. Otherwise seek to the previos one.
OD_LOG("SeekTime: %lld, mLastSeekTime:%lld", aTimeUs, mLastSeekTime);
if (mLastSeekTime == -1 || mLastSeekTime > aTimeUs) {
seekMode = MediaSource::ReadOptions::SEEK_PREVIOUS_SYNC;
} else {
seekMode = MediaSource::ReadOptions::SEEK_NEXT_SYNC;
}
mLastSeekTime = aTimeUs;
bool findNextBuffer = true;
while (findNextBuffer) {
options.setSeekTo(aTimeUs, seekMode);
findNextBuffer = false;
if (mIsVideoSeeking) {
err = mVideoSource->read(&mVideoBuffer, &options);
Mutex::Autolock autoLock(mSeekLock);
mIsVideoSeeking = false;
PostReleaseVideoBuffer(nullptr, FenceHandle());
}
else {
err = mVideoSource->read(&mVideoBuffer);
}
// If there is no next Keyframe, jump to the previous key frame.
if (err == ERROR_END_OF_STREAM && seekMode == MediaSource::ReadOptions::SEEK_NEXT_SYNC) {
seekMode = MediaSource::ReadOptions::SEEK_PREVIOUS_SYNC;
findNextBuffer = true;
{
Mutex::Autolock autoLock(mSeekLock);
mIsVideoSeeking = true;
}
continue;
} else if (err != OK) {
OD_LOG("Unexpected error when seeking to %lld", aTimeUs);
break;
}
// For some codecs, the length of first decoded frame after seek is 0.
// Need to ignore it and continue to find the next one
if (mVideoBuffer->range_length() == 0) {
PostReleaseVideoBuffer(mVideoBuffer, FenceHandle());
findNextBuffer = true;
}
}
aDoSeek = false;
} else {
err = mVideoSource->read(&mVideoBuffer);
}
aFrame->mSize = 0;
if (err == OK) {
int64_t timeUs;
int32_t unreadable;
int32_t keyFrame;
size_t length = mVideoBuffer->range_length();
if (!mVideoBuffer->meta_data()->findInt64(kKeyTime, &timeUs) ) {
NS_WARNING("OMX decoder did not return frame time");
return false;
}
if (!mVideoBuffer->meta_data()->findInt32(kKeyIsSyncFrame, &keyFrame)) {
keyFrame = 0;
}
if (!mVideoBuffer->meta_data()->findInt32(kKeyIsUnreadable, &unreadable)) {
unreadable = 0;
}
RefPtr<mozilla::layers::TextureClient> textureClient;
if ((mVideoBuffer->graphicBuffer().get())) {
textureClient = mNativeWindow->getTextureClientFromBuffer(mVideoBuffer->graphicBuffer().get());
}
if (textureClient) {
// Manually increment reference count to keep MediaBuffer alive
// during TextureClient is in use.
mVideoBuffer->add_ref();
static_cast<GrallocTextureData*>(textureClient->GetInternalData())->SetMediaBuffer(mVideoBuffer);
// Set recycle callback for TextureClient
textureClient->SetRecycleCallback(OmxDecoder::RecycleCallback, this);
{
Mutex::Autolock autoLock(mPendingVideoBuffersLock);
// Store pending recycle TextureClient.
MOZ_ASSERT(mPendingRecycleTexutreClients.find(textureClient) == mPendingRecycleTexutreClients.end());
mPendingRecycleTexutreClients.insert(textureClient);
}
aFrame->mGraphicBuffer = textureClient;
aFrame->mRotation = mVideoRotation;
aFrame->mTimeUs = timeUs;
aFrame->mKeyFrame = keyFrame;
aFrame->Y.mWidth = mVideoWidth;
aFrame->Y.mHeight = mVideoHeight;
// Release to hold video buffer in OmxDecoder more.
// MediaBuffer's ref count is changed from 2 to 1.
ReleaseVideoBuffer();
} else if (length > 0) {
char *data = static_cast<char *>(mVideoBuffer->data()) + mVideoBuffer->range_offset();
if (unreadable) {
LOG(LogLevel::Debug, "video frame is unreadable");
}
if (!ToVideoFrame(aFrame, timeUs, data, length, keyFrame)) {
return false;
}
}
// Check if this frame is valid or not. If not, skip it.
if ((aKeyframeSkip && timeUs < aTimeUs) || length == 0) {
aFrame->mShouldSkip = true;
}
}
else if (err == INFO_FORMAT_CHANGED) {
// If the format changed, update our cached info.
if (!SetVideoFormat()) {
return false;
} else {
return ReadVideo(aFrame, aTimeUs, aKeyframeSkip, aDoSeek);
}
}
else if (err == ERROR_END_OF_STREAM) {
return false;
}
else if (err == -ETIMEDOUT) {
LOG(LogLevel::Debug, "OmxDecoder::ReadVideo timed out, will retry");
return true;
}
else {
// UNKNOWN_ERROR is sometimes is used to mean "out of memory", but
// regardless, don't keep trying to decode if the decoder doesn't want to.
LOG(LogLevel::Debug, "OmxDecoder::ReadVideo failed, err=%d", err);
return false;
}
return true;
}
size_t AudioOffloadPlayer::FillBuffer(void* aData, size_t aSize)
{
CHECK(mAudioSink.get());
if (mReachedEOS) {
return 0;
}
size_t sizeDone = 0;
size_t sizeRemaining = aSize;
int64_t seekTimeUs = -1;
while (sizeRemaining > 0) {
MediaSource::ReadOptions options;
bool refreshSeekTime = false;
{
android::Mutex::Autolock autoLock(mLock);
if (mSeekTarget.IsValid()) {
seekTimeUs = mSeekTarget.mTime;
options.setSeekTo(seekTimeUs);
refreshSeekTime = true;
if (mInputBuffer) {
mInputBuffer->release();
mInputBuffer = nullptr;
}
}
}
if (!mInputBuffer) {
status_t err;
err = mSource->read(&mInputBuffer, &options);
CHECK((!err && mInputBuffer) || (err && !mInputBuffer));
android::Mutex::Autolock autoLock(mLock);
if (err != OK) {
if (mSeekTarget.IsValid()) {
mSeekTarget.Reset();
}
AUDIO_OFFLOAD_LOG(PR_LOG_ERROR, ("Error while reading media source %d "
"Ok to receive EOS error at end", err));
if (!mReachedEOS) {
// After seek there is a possible race condition if
// OffloadThread is observing state_stopping_1 before
// framesReady() > 0. Ensure sink stop is called
// after last buffer is released. This ensures the
// partial buffer is written to the driver before
// stopping one is observed.The drawback is that
// there will be an unnecessary call to the parser
// after parser signalled EOS.
if (sizeDone > 0) {
AUDIO_OFFLOAD_LOG(PR_LOG_DEBUG, ("send Partial buffer down"));
AUDIO_OFFLOAD_LOG(PR_LOG_DEBUG, ("skip calling stop till next"
" fillBuffer"));
break;
}
// no more buffers to push - stop() and wait for STREAM_END
// don't set mReachedEOS until stream end received
mAudioSink->Stop();
}
break;
}
if(mInputBuffer->range_length() != 0) {
CHECK(mInputBuffer->meta_data()->findInt64(
kKeyTime, &mPositionTimeMediaUs));
}
if (mSeekTarget.IsValid() && seekTimeUs == mSeekTarget.mTime) {
MOZ_ASSERT(mSeekTarget.IsValid());
mSeekTarget.Reset();
if (!mSeekPromise.IsEmpty()) {
AUDIO_OFFLOAD_LOG(PR_LOG_DEBUG, ("FillBuffer posting SEEK_COMPLETE"));
MediaDecoder::SeekResolveValue val(mReachedEOS, mSeekTarget.mEventVisibility);
mSeekPromise.Resolve(val, __func__);
}
} else if (mSeekTarget.IsValid()) {
AUDIO_OFFLOAD_LOG(PR_LOG_DEBUG, ("seek is updated during unlocking mLock"));
}
if (refreshSeekTime) {
NotifyPositionChanged();
// need to adjust the mStartPosUs for offload decoding since parser
// might not be able to get the exact seek time requested.
mStartPosUs = mPositionTimeMediaUs;
AUDIO_OFFLOAD_LOG(PR_LOG_DEBUG, ("Adjust seek time to: %.2f",
mStartPosUs / 1E6));
}
}
if (mInputBuffer->range_length() == 0) {
mInputBuffer->release();
mInputBuffer = nullptr;
continue;
}
size_t copy = sizeRemaining;
if (copy > mInputBuffer->range_length()) {
copy = mInputBuffer->range_length();
}
memcpy((char *)aData + sizeDone,
(const char *)mInputBuffer->data() + mInputBuffer->range_offset(),
copy);
mInputBuffer->set_range(mInputBuffer->range_offset() + copy,
mInputBuffer->range_length() - copy);
sizeDone += copy;
sizeRemaining -= copy;
}
return sizeDone;
}
status_t AudioPlayer::start(bool sourceAlreadyStarted) {
CHECK(!mStarted);
CHECK(mSource != NULL);
status_t err;
if (!sourceAlreadyStarted) {
#ifdef QCOM_HARDWARE
mSourcePaused = false;
#endif
err = mSource->start();
if (err != OK) {
return err;
}
}
// We allow an optional INFO_FORMAT_CHANGED at the very beginning
// of playback, if there is one, getFormat below will retrieve the
// updated format, if there isn't, we'll stash away the valid buffer
// of data to be used on the first audio callback.
CHECK(mFirstBuffer == NULL);
MediaSource::ReadOptions options;
if (mSeeking) {
options.setSeekTo(mSeekTimeUs);
mSeeking = false;
}
mFirstBufferResult = mSource->read(&mFirstBuffer, &options);
if (mFirstBufferResult == INFO_FORMAT_CHANGED) {
LOGV("INFO_FORMAT_CHANGED!!!");
CHECK(mFirstBuffer == NULL);
mFirstBufferResult = OK;
mIsFirstBuffer = false;
} else {
mIsFirstBuffer = true;
}
sp<MetaData> format = mSource->getFormat();
const char *mime;
bool success = format->findCString(kKeyMIMEType, &mime);
CHECK(success);
CHECK(!strcasecmp(mime, MEDIA_MIMETYPE_AUDIO_RAW));
success = format->findInt32(kKeySampleRate, &mSampleRate);
CHECK(success);
int32_t numChannels;
success = format->findInt32(kKeyChannelCount, &numChannels);
CHECK(success);
if (mAudioSink.get() != NULL) {
status_t err = mAudioSink->open(
mSampleRate, numChannels, AUDIO_FORMAT_PCM_16_BIT,
DEFAULT_AUDIOSINK_BUFFERCOUNT,
#ifdef STE_HARDWARE
&AudioPlayer::AudioSinkCallback, this,
&AudioPlayer::LatencyCallback);
#else
&AudioPlayer::AudioSinkCallback, this);
#endif
if (err != OK) {
if (mFirstBuffer != NULL) {
mFirstBuffer->release();
mFirstBuffer = NULL;
}
if (!sourceAlreadyStarted) {
mSource->stop();
}
return err;
}
mLatencyUs = (int64_t)mAudioSink->latency() * 1000;
mFrameSize = mAudioSink->frameSize();
mAudioSink->start();
} else {
bool OmxDecoder::ReadVideo(VideoFrame *aFrame, int64_t aTimeUs,
bool aKeyframeSkip, bool aDoSeek)
{
if (!mVideoSource.get())
return false;
ReleaseVideoBuffer();
status_t err;
if (aDoSeek) {
{
Mutex::Autolock autoLock(mSeekLock);
mIsVideoSeeking = true;
}
MediaSource::ReadOptions options;
options.setSeekTo(aTimeUs, MediaSource::ReadOptions::SEEK_PREVIOUS_SYNC);
err = mVideoSource->read(&mVideoBuffer, &options);
{
Mutex::Autolock autoLock(mSeekLock);
mIsVideoSeeking = false;
ReleaseAllPendingVideoBuffersLocked();
}
} else {
err = mVideoSource->read(&mVideoBuffer);
}
if (err == OK && mVideoBuffer->range_length() > 0) {
int64_t timeUs;
int64_t durationUs;
int32_t unreadable;
int32_t keyFrame;
if (!mVideoBuffer->meta_data()->findInt64(kKeyTime, &timeUs) ) {
NS_WARNING("OMX decoder did not return frame time");
return false;
}
if (!mVideoBuffer->meta_data()->findInt32(kKeyIsSyncFrame, &keyFrame)) {
keyFrame = 0;
}
if (!mVideoBuffer->meta_data()->findInt32(kKeyIsUnreadable, &unreadable)) {
unreadable = 0;
}
mozilla::layers::SurfaceDescriptor *descriptor = nullptr;
if ((mVideoBuffer->graphicBuffer().get())) {
descriptor = mNativeWindow->getSurfaceDescriptorFromBuffer(mVideoBuffer->graphicBuffer().get());
}
if (descriptor) {
// Change the descriptor's size to video's size. There are cases that
// GraphicBuffer's size and actual video size is different.
// See Bug 850566.
const mozilla::layers::SurfaceDescriptorGralloc& grallocDesc = descriptor->get_SurfaceDescriptorGralloc();
mozilla::layers::SurfaceDescriptor newDescriptor = mozilla::layers::SurfaceDescriptorGralloc(grallocDesc.bufferParent(),
grallocDesc.bufferChild(), nsIntSize(mVideoWidth, mVideoHeight), grallocDesc.external());
aFrame->mGraphicBuffer = new mozilla::layers::VideoGraphicBuffer(this, mVideoBuffer, &newDescriptor);
aFrame->mRotation = mVideoRotation;
aFrame->mTimeUs = timeUs;
aFrame->mEndTimeUs = timeUs + durationUs;
aFrame->mKeyFrame = keyFrame;
aFrame->Y.mWidth = mVideoWidth;
aFrame->Y.mHeight = mVideoHeight;
} else {
char *data = static_cast<char *>(mVideoBuffer->data()) + mVideoBuffer->range_offset();
size_t length = mVideoBuffer->range_length();
if (unreadable) {
LOG(PR_LOG_DEBUG, "video frame is unreadable");
}
if (!ToVideoFrame(aFrame, timeUs, data, length, keyFrame)) {
return false;
}
aFrame->mEndTimeUs = timeUs + durationUs;
}
if (aKeyframeSkip && timeUs < aTimeUs) {
aFrame->mShouldSkip = true;
}
}
else if (err == INFO_FORMAT_CHANGED) {
// If the format changed, update our cached info.
if (!SetVideoFormat()) {
return false;
} else {
return ReadVideo(aFrame, aTimeUs, aKeyframeSkip, aDoSeek);
}
}
else if (err == ERROR_END_OF_STREAM) {
return false;
}
else if (err == UNKNOWN_ERROR) {
// This sometimes is used to mean "out of memory", but regardless,
// don't keep trying to decode if the decoder doesn't want to.
return false;
}
return true;
}
size_t AudioOffloadPlayer::FillBuffer(void* aData, size_t aSize)
{
CHECK(mAudioSink.get());
if (mReachedEOS) {
return 0;
}
size_t sizeDone = 0;
size_t sizeRemaining = aSize;
while (sizeRemaining > 0) {
MediaSource::ReadOptions options;
bool refreshSeekTime = false;
{
android::Mutex::Autolock autoLock(mLock);
if (mSeeking) {
options.setSeekTo(mSeekTimeUs);
refreshSeekTime = true;
if (mInputBuffer) {
mInputBuffer->release();
mInputBuffer = nullptr;
}
mSeeking = false;
}
}
if (!mInputBuffer) {
status_t err;
err = mSource->read(&mInputBuffer, &options);
CHECK((!err && mInputBuffer) || (err && !mInputBuffer));
android::Mutex::Autolock autoLock(mLock);
if (err != OK) {
AUDIO_OFFLOAD_LOG(PR_LOG_ERROR, ("Error while reading media source %d "
"Ok to receive EOS error at end", err));
if (!mReachedEOS) {
// After seek there is a possible race condition if
// OffloadThread is observing state_stopping_1 before
// framesReady() > 0. Ensure sink stop is called
// after last buffer is released. This ensures the
// partial buffer is written to the driver before
// stopping one is observed.The drawback is that
// there will be an unnecessary call to the parser
// after parser signalled EOS.
if (sizeDone > 0) {
AUDIO_OFFLOAD_LOG(PR_LOG_DEBUG, ("send Partial buffer down"));
AUDIO_OFFLOAD_LOG(PR_LOG_DEBUG, ("skip calling stop till next"
" fillBuffer"));
break;
}
// no more buffers to push - stop() and wait for STREAM_END
// don't set mReachedEOS until stream end received
mAudioSink->Stop();
}
break;
}
if(mInputBuffer->range_length() != 0) {
CHECK(mInputBuffer->meta_data()->findInt64(
kKeyTime, &mPositionTimeMediaUs));
}
if (refreshSeekTime) {
if (mDispatchSeekEvents && !mSeekDuringPause) {
mDispatchSeekEvents = false;
AUDIO_OFFLOAD_LOG(PR_LOG_DEBUG, ("FillBuffer posting SEEK_COMPLETE"));
nsCOMPtr<nsIRunnable> nsEvent = NS_NewRunnableMethod(mObserver,
&MediaDecoder::SeekingStopped);
NS_DispatchToMainThread(nsEvent, NS_DISPATCH_NORMAL);
} else if (mSeekDuringPause) {
// Callback is already called for seek during pause. Just reset the
// flag
AUDIO_OFFLOAD_LOG(PR_LOG_DEBUG, ("Not posting seek complete as its"
" already faked"));
mSeekDuringPause = false;
}
NotifyPositionChanged();
// need to adjust the mStartPosUs for offload decoding since parser
// might not be able to get the exact seek time requested.
mStartPosUs = mPositionTimeMediaUs;
AUDIO_OFFLOAD_LOG(PR_LOG_DEBUG, ("Adjust seek time to: %.2f",
mStartPosUs / 1E6));
// clear seek time with mLock locked and once we have valid
// mPositionTimeMediaUs
// before clearing mSeekTimeUs check if a new seek request has been
// received while we were reading from the source with mLock released.
if (!mSeeking) {
mSeekTimeUs = 0;
}
}
}
if (mInputBuffer->range_length() == 0) {
mInputBuffer->release();
mInputBuffer = nullptr;
continue;
}
size_t copy = sizeRemaining;
if (copy > mInputBuffer->range_length()) {
copy = mInputBuffer->range_length();
}
memcpy((char *)aData + sizeDone,
(const char *)mInputBuffer->data() + mInputBuffer->range_offset(),
copy);
mInputBuffer->set_range(mInputBuffer->range_offset() + copy,
mInputBuffer->range_length() - copy);
sizeDone += copy;
sizeRemaining -= copy;
}
return sizeDone;
}
status_t TunnelPlayer::start(bool sourceAlreadyStarted) {
CHECK(!mStarted);
CHECK(mSource != NULL);
ALOGV("start: sourceAlreadyStarted %d", sourceAlreadyStarted);
//Check if the source is started, start it
status_t err;
if (!sourceAlreadyStarted) {
err = mSource->start();
if (err != OK) {
return err;
}
}
//Create extractor thread, read and initialize all the
//mutexes and coditional variables
createThreads();
ALOGV("Thread Created.");
// We allow an optional INFO_FORMAT_CHANGED at the very beginning
// of playback, if there is one, getFormat below will retrieve the
// updated format, if there isn't, we'll stash away the valid buffer
// of data to be used on the first audio callback.
CHECK(mFirstBuffer == NULL);
MediaSource::ReadOptions options;
if (mSeeking) {
options.setSeekTo(mSeekTimeUs);
mSeeking = false;
}
mFirstBufferResult = mSource->read(&mFirstBuffer, &options);
if (mFirstBufferResult == INFO_FORMAT_CHANGED) {
ALOGV("INFO_FORMAT_CHANGED!!!");
CHECK(mFirstBuffer == NULL);
mFirstBufferResult = OK;
mIsFirstBuffer = false;
} else {
mIsFirstBuffer = true;
}
sp<MetaData> format = mSource->getFormat();
const char *mime;
bool success = format->findCString(kKeyMIMEType, &mime);
if (!strcasecmp(mime,MEDIA_MIMETYPE_AUDIO_AAC)) {
mFormat = AUDIO_FORMAT_AAC;
}
else if (!strcasecmp(mime,MEDIA_MIMETYPE_AUDIO_MPEG)) {
mFormat = AUDIO_FORMAT_MP3;
ALOGD("TunnelPlayer::start AUDIO_FORMAT_MP3");
} else {
ALOGE("TunnelPlayer::UNSUPPORTED");
}
CHECK(success);
success = format->findInt32(kKeySampleRate, &mSampleRate);
CHECK(success);
success = format->findInt32(kKeyChannelCount, &numChannels);
CHECK(success);
if(!format->findInt32(kKeyChannelMask, &mChannelMask)) {
// log only when there's a risk of ambiguity of channel mask selection
ALOGI_IF(numChannels > 2,
"source format didn't specify channel mask, using (%d) channel order", numChannels);
mChannelMask = CHANNEL_MASK_USE_CHANNEL_ORDER;
}
audio_output_flags_t flags = (audio_output_flags_t) (AUDIO_OUTPUT_FLAG_TUNNEL |
AUDIO_OUTPUT_FLAG_DIRECT);
ALOGV("mAudiosink->open() mSampleRate %d, numChannels %d, mChannelMask %d, flags %d",mSampleRate, numChannels, mChannelMask, flags);
err = mAudioSink->open(
mSampleRate, numChannels, mChannelMask, mFormat,
DEFAULT_AUDIOSINK_BUFFERCOUNT,
&TunnelPlayer::AudioSinkCallback,
this,
flags,
NULL);
if (err != OK) {
if (mFirstBuffer != NULL) {
mFirstBuffer->release();
mFirstBuffer = NULL;
}
if (!sourceAlreadyStarted) {
mSource->stop();
}
ALOGE("Opening a routing session failed");
return err;
}
mIsAudioRouted = true;
mStarted = true;
mAudioSink->start();
mLock.lock();
ALOGV("Waking up extractor thread");
mExtractorCv.signal();
mLock.unlock();
return OK;
}
static VideoFrame *extractVideoFrameWithCodecFlags(
OMXClient *client,
const sp<MetaData> &trackMeta,
const sp<MediaSource> &source,
uint32_t flags,
int64_t frameTimeUs,
int seekMode) {
#ifdef OMAP_ENHANCEMENT
flags |= OMXCodec::kPreferThumbnailMode;
#ifdef TARGET_OMAP4
int32_t isInterlaced = false;
//Call config parser to update profile,level,interlaced,reference frame data
updateMetaData(trackMeta);
trackMeta->findInt32(kKeyVideoInterlaced, &isInterlaced);
if(isInterlaced)
{
flags |= OMXCodec::kPreferInterlacedOutputContent;
}
#endif
#endif
sp<MediaSource> decoder =
OMXCodec::Create(
client->interface(), source->getFormat(), false, source,
NULL, flags | OMXCodec::kClientNeedsFramebuffer);
if (decoder.get() == NULL) {
LOGV("unable to instantiate video decoder.");
return NULL;
}
status_t err = decoder->start();
if (err != OK) {
LOGW("OMXCodec::start returned error %d (0x%08x)\n", err, err);
return NULL;
}
// Read one output buffer, ignore format change notifications
// and spurious empty buffers.
MediaSource::ReadOptions options;
if (seekMode < MediaSource::ReadOptions::SEEK_PREVIOUS_SYNC ||
seekMode > MediaSource::ReadOptions::SEEK_CLOSEST) {
LOGE("Unknown seek mode: %d", seekMode);
return NULL;
}
MediaSource::ReadOptions::SeekMode mode =
static_cast<MediaSource::ReadOptions::SeekMode>(seekMode);
int64_t thumbNailTime;
if (frameTimeUs < 0 && trackMeta->findInt64(kKeyThumbnailTime, &thumbNailTime)) {
options.setSeekTo(thumbNailTime, mode);
} else {
thumbNailTime = -1;
options.setSeekTo(frameTimeUs < 0 ? 0 : frameTimeUs, mode);
}
MediaBuffer *buffer = NULL;
do {
if (buffer != NULL) {
buffer->release();
buffer = NULL;
}
err = decoder->read(&buffer, &options);
#ifdef OMAP_ENHANCEMENT
if(err == INFO_FORMAT_CHANGED)
{
int32_t w1,h1;
decoder->getFormat()->findInt32(kKeyWidth, &w1);
decoder->getFormat()->findInt32(kKeyHeight, &h1);
LOGD("Got portreconfig event. New WxH %dx%d. wait 5mS for port to be enabled",w1,h1);
usleep(5000); //sleep 5mS for port disable-enable to complete
}
#endif
options.clearSeekTo();
} while (err == INFO_FORMAT_CHANGED
|| (buffer != NULL && buffer->range_length() == 0));
if (err != OK) {
CHECK_EQ(buffer, NULL);
LOGV("decoding frame failed.");
decoder->stop();
return NULL;
}
LOGV("successfully decoded video frame.");
int32_t unreadable;
if (buffer->meta_data()->findInt32(kKeyIsUnreadable, &unreadable)
&& unreadable != 0) {
LOGV("video frame is unreadable, decoder does not give us access "
"to the video data.");
buffer->release();
buffer = NULL;
decoder->stop();
return NULL;
}
int64_t timeUs;
CHECK(buffer->meta_data()->findInt64(kKeyTime, &timeUs));
if (thumbNailTime >= 0) {
if (timeUs != thumbNailTime) {
const char *mime;
CHECK(trackMeta->findCString(kKeyMIMEType, &mime));
LOGV("thumbNailTime = %lld us, timeUs = %lld us, mime = %s",
thumbNailTime, timeUs, mime);
}
}
sp<MetaData> meta = decoder->getFormat();
int32_t width, height;
CHECK(meta->findInt32(kKeyWidth, &width));
CHECK(meta->findInt32(kKeyHeight, &height));
int32_t rotationAngle;
if (!trackMeta->findInt32(kKeyRotation, &rotationAngle)) {
rotationAngle = 0; // By default, no rotation
}
VideoFrame *frame = new VideoFrame;
#if defined(OMAP_ENHANCEMENT) && defined(TARGET_OMAP4)
int32_t srcFormat;
CHECK(meta->findInt32(kKeyColorFormat, &srcFormat));
int32_t format;
const char *component;
//cache the display width and height
int32_t displayWidth, displayHeight;
displayWidth = width;
displayHeight = height;
//update width & height with the buffer width&height
if(!(meta->findInt32(kKeyPaddedWidth, &width))) {
CHECK(meta->findInt32(kKeyWidth, &width));
}
if(!(meta->findInt32(kKeyPaddedHeight, &height))) {
CHECK(meta->findInt32(kKeyHeight, &height));
}
LOGD("VideoFrame WxH %dx%d", displayWidth, displayHeight);
if(((OMX_COLOR_FORMATTYPE)srcFormat == OMX_COLOR_FormatYUV420PackedSemiPlanar) ||
((OMX_COLOR_FORMATTYPE)srcFormat == OMX_TI_COLOR_FormatYUV420PackedSemiPlanar_Sequential_TopBottom)){
frame->mWidth = displayWidth;
frame->mHeight = displayHeight;
frame->mDisplayWidth = displayWidth;
frame->mDisplayHeight = displayHeight;
frame->mSize = displayWidth * displayHeight * 2;
frame->mData = new uint8_t[frame->mSize];
frame->mRotationAngle = rotationAngle;
}else {
frame->mWidth = width;
frame->mHeight = height;
frame->mDisplayWidth = width;
frame->mDisplayHeight = height;
frame->mSize = width * height * 2;
frame->mData = new uint8_t[frame->mSize];
frame->mRotationAngle = rotationAngle;
}
if(((OMX_COLOR_FORMATTYPE)srcFormat == OMX_COLOR_FormatYUV420PackedSemiPlanar) ||
((OMX_COLOR_FORMATTYPE)srcFormat == OMX_TI_COLOR_FormatYUV420PackedSemiPlanar_Sequential_TopBottom)){
ColorConverter converter(
(OMX_COLOR_FORMATTYPE)srcFormat, OMX_COLOR_Format16bitRGB565);
CHECK(converter.isValid());
converter.convert(
width, height,
(const uint8_t *)buffer->data() + buffer->range_offset(),
0, //1D buffer in 1.16 Ducati rls. If 2D buffer -> 4096 stride should be used
frame->mData, displayWidth * 2,
displayWidth,displayHeight,buffer->range_offset(),isInterlaced);
}
else{
ColorConverter converter(
(OMX_COLOR_FORMATTYPE)srcFormat, OMX_COLOR_Format16bitRGB565);
CHECK(converter.isValid());
converter.convert(
width, height,
(const uint8_t *)buffer->data() + buffer->range_offset(),
0,
frame->mData, width * 2);
}
#else
frame->mWidth = width;
frame->mHeight = height;
frame->mDisplayWidth = width;
frame->mDisplayHeight = height;
frame->mSize = width * height * 2;
frame->mData = new uint8_t[frame->mSize];
frame->mRotationAngle = rotationAngle;
int32_t srcFormat;
CHECK(meta->findInt32(kKeyColorFormat, &srcFormat));
ColorConverter converter(
(OMX_COLOR_FORMATTYPE)srcFormat, OMX_COLOR_Format16bitRGB565);
CHECK(converter.isValid());
converter.convert(
width, height,
(const uint8_t *)buffer->data() + buffer->range_offset(),
0,
frame->mData, width * 2);
#endif
buffer->release();
buffer = NULL;
decoder->stop();
return frame;
}
/*!
* \brief The decode thread function.
* The main Loop is used to read data from decoder, and put them to render.
*/
void UMMediaPlayer::videoEntry(void)
{
bool eof = false;
MediaBuffer *lastBuffer = NULL;
MediaBuffer *buffer;
MediaSource::ReadOptions options;
mVideoStartTime = 0;
dcount =0;
UMLOG_ERR("videoEntry() ---- mVideoDecoder->start() begin");
status_t err = mVideoDecoder->start();
if(err != OK)
{
UMLOG_ERR("videoEntry() ---- mVideoDecoder->start() end failed err=%d", err);
mHasDspError = 1;
mPlaying = false;
if(mVideoSource->HasAnyDataSource())
{
mVideoSource->read(&buffer, &options);
releaseBufferIfNonNULL(&buffer);
}
}
while(mPlaying && !mVideoSource->HasAnyDataSource())
{
usleep(50 * 1000);
}
while(mPlaying)
{
status_t err = mVideoDecoder->read(&buffer, &options);
options.clearSeekTo();
if(err == INFO_FORMAT_CHANGED)
{
UMLOG_ERR("VideoSource signalled format change.");
if(mVideoRenderer != NULL)
{
initRenderer();
}
continue;
}
if(err != OK && buffer == NULL)
{
mHasDspError = 1;
mPlaying = false;
continue;
}
dcount++;
printPDecodeDateToFile((char*)buffer->data());
if(dcount == 10){
if(fd != NULL){
fclose(fd);
}
}
if(buffer == NULL)
{
usleep(3000);
eof = true;
continue;
}
CHECK((err == OK && buffer != NULL) || (err != OK && buffer == NULL));
if(err != OK)
{
eof = true;
mPlaying = false;
continue;
}
if(buffer->range_length() == 0)
{
buffer->release();
buffer = NULL;
continue;
}
int64_t timeUs;
CHECK(buffer->meta_data()->findInt64(kKeyTime, &timeUs));
if(0 == timeUs)
{
unsigned int currTS = mVideoSource->um_util_getCurrentTick();
if(currTS <= lastRenderTS + UM_RENDER_MAX_INTERVAL)
{
//There is a frame has been pushed into render in last UM_RENDER_MAX_INTERVAL ms
//skip the compensate frame then
buffer->release();
buffer = NULL;
continue;
}
/* push componsate frame to render */
}
displayOrDiscardFrame(&lastBuffer, buffer, 0);
}
releaseBufferIfNonNULL(&lastBuffer);
shutdownVideoDecoder();
}
size_t AudioPlayer::fillBuffer(void *data, size_t size) {
if (mNumFramesPlayed == 0) {
ALOGV("AudioCallback");
}
if (mReachedEOS) {
return 0;
}
bool postSeekComplete = false;
bool postEOS = false;
int64_t postEOSDelayUs = 0;
size_t size_done = 0;
size_t size_remaining = size;
while (size_remaining > 0) {
MediaSource::ReadOptions options;
{
Mutex::Autolock autoLock(mLock);
if (mSeeking) {
if (mIsFirstBuffer) {
if (mFirstBuffer != NULL) {
mFirstBuffer->release();
mFirstBuffer = NULL;
}
mIsFirstBuffer = false;
}
options.setSeekTo(mSeekTimeUs);
if (mInputBuffer != NULL) {
mInputBuffer->release();
mInputBuffer = NULL;
}
mSeeking = false;
if (mObserver) {
postSeekComplete = true;
}
}
}
if (mInputBuffer == NULL) {
status_t err;
if (mIsFirstBuffer) {
mInputBuffer = mFirstBuffer;
mFirstBuffer = NULL;
err = mFirstBufferResult;
mIsFirstBuffer = false;
} else {
err = mSource->read(&mInputBuffer, &options);
#ifdef QCOM_HARDWARE
if (err == OK && mInputBuffer == NULL && mSourcePaused) {
ALOGV("mSourcePaused, return 0 from fillBuffer");
return 0;
}
#endif
}
CHECK((err == OK && mInputBuffer != NULL)
|| (err != OK && mInputBuffer == NULL));
Mutex::Autolock autoLock(mLock);
if (err != OK) {
if (mObserver && !mReachedEOS) {
// We don't want to post EOS right away but only
// after all frames have actually been played out.
// These are the number of frames submitted to the
// AudioTrack that you haven't heard yet.
uint32_t numFramesPendingPlayout =
getNumFramesPendingPlayout();
// These are the number of frames we're going to
// submit to the AudioTrack by returning from this
// callback.
uint32_t numAdditionalFrames = size_done / mFrameSize;
numFramesPendingPlayout += numAdditionalFrames;
int64_t timeToCompletionUs =
(1000000ll * numFramesPendingPlayout) / mSampleRate;
ALOGV("total number of frames played: %lld (%lld us)",
(mNumFramesPlayed + numAdditionalFrames),
1000000ll * (mNumFramesPlayed + numAdditionalFrames)
/ mSampleRate);
ALOGV("%d frames left to play, %lld us (%.2f secs)",
numFramesPendingPlayout,
timeToCompletionUs, timeToCompletionUs / 1E6);
postEOS = true;
if (mAudioSink->needsTrailingPadding()) {
postEOSDelayUs = timeToCompletionUs + mLatencyUs;
} else {
postEOSDelayUs = 0;
}
}
mReachedEOS = true;
mFinalStatus = err;
break;
}
if (mAudioSink != NULL) {
mLatencyUs = (int64_t)mAudioSink->latency() * 1000;
} else {
mLatencyUs = (int64_t)mAudioTrack->latency() * 1000;
}
CHECK(mInputBuffer->meta_data()->findInt64(
kKeyTime, &mPositionTimeMediaUs));
mPositionTimeRealUs =
((mNumFramesPlayed + size_done / mFrameSize) * 1000000)
/ mSampleRate;
ALOGV("buffer->size() = %d, "
"mPositionTimeMediaUs=%.2f mPositionTimeRealUs=%.2f",
mInputBuffer->range_length(),
mPositionTimeMediaUs / 1E6, mPositionTimeRealUs / 1E6);
}
if (mInputBuffer->range_length() == 0) {
mInputBuffer->release();
mInputBuffer = NULL;
continue;
}
size_t copy = size_remaining;
if (copy > mInputBuffer->range_length()) {
copy = mInputBuffer->range_length();
}
memcpy((char *)data + size_done,
(const char *)mInputBuffer->data() + mInputBuffer->range_offset(),
copy);
mInputBuffer->set_range(mInputBuffer->range_offset() + copy,
mInputBuffer->range_length() - copy);
size_done += copy;
size_remaining -= copy;
}
{
Mutex::Autolock autoLock(mLock);
mNumFramesPlayed += size_done / mFrameSize;
mNumFramesPlayedSysTimeUs = ALooper::GetNowUs();
if (mReachedEOS) {
mPinnedTimeUs = mNumFramesPlayedSysTimeUs;
} else {
mPinnedTimeUs = -1ll;
}
}
if (postEOS) {
mObserver->postAudioEOS(postEOSDelayUs);
}
if (postSeekComplete) {
mObserver->postAudioSeekComplete();
}
return size_done;
}
size_t TunnelPlayer::fillBuffer(void *data, size_t size) {
if (mReachedEOS) {
return 0;
}
bool postSeekComplete = false;
size_t size_done = 0;
size_t size_remaining = size;
//clear the flag since we dont know whether we are seeking or not, yet
ALOGV("fillBuffer: Clearing seek flag in fill buffer");
bool yield = !mIsFirstBuffer;
while (size_remaining > 0) {
MediaSource::ReadOptions options;
{
Mutex::Autolock autoLock(mLock);
if(mSeeking) {
mInternalSeeking = false;
}
if (mSeeking || mInternalSeeking) {
if (mIsFirstBuffer) {
if (mFirstBuffer != NULL) {
mFirstBuffer->release();
mFirstBuffer = NULL;
}
mIsFirstBuffer = false;
}
MediaSource::ReadOptions::SeekMode seekMode;
seekMode = MediaSource::ReadOptions::SEEK_CLOSEST_SYNC;
options.setSeekTo(mSeekTimeUs, seekMode );
if (mInputBuffer != NULL) {
mInputBuffer->release();
mInputBuffer = NULL;
}
// This is to ignore the data already filled in the output buffer
size_done = 0;
size_remaining = size;
mSeeking = false;
if (mObserver && !mInternalSeeking) {
ALOGV("fillBuffer: Posting audio seek complete event");
postSeekComplete = true;
}
mInternalSeeking = false;
ALOGV("fillBuffer: Setting seek flag in fill buffer");
//set the flag since we know that this buffer is the new positions buffer
}
}
if (mInputBuffer == NULL) {
status_t err;
if (mIsFirstBuffer) {
mInputBuffer = mFirstBuffer;
mFirstBuffer = NULL;
err = mFirstBufferResult;
mIsFirstBuffer = false;
} else {
err = mSource->read(&mInputBuffer, &options);
}
CHECK((err == OK && mInputBuffer != NULL)
|| (err != OK && mInputBuffer == NULL));
{
Mutex::Autolock autoLock(mLock);
if (err != OK) {
ALOGD("fill buffer - reached eos true");
mReachedEOS = true;
mFinalStatus = err;
break;
}
}
}
if (mInputBuffer->range_length() == 0) {
mInputBuffer->release();
mInputBuffer = NULL;
continue;
}
size_t copy = size_remaining;
if (copy > mInputBuffer->range_length()) {
copy = mInputBuffer->range_length();
}
memcpy((char *)data + size_done,
(const char *)mInputBuffer->data() + mInputBuffer->range_offset(),
copy);
mInputBuffer->set_range(mInputBuffer->range_offset() + copy,
mInputBuffer->range_length() - copy);
size_done += copy;
size_remaining -= copy;
if (mHasVideo && yield) {
sched_yield();
}
}
if(mReachedEOS)
memset((char *)data + size_done, 0x0, size_remaining);
ALOGV("fill buffer size_done = %d",size_done);
if (postSeekComplete) {
mObserver->postAudioSeekComplete();
}
return size_done;
}
size_t AudioPlayer::fillBuffer(void *data, size_t size) {
if (mNumFramesPlayed == 0) {
LOGV("AudioCallback");
}
if (mReachedEOS) {
return 0;
}
size_t size_done = 0;
size_t size_remaining = size;
while (size_remaining > 0) {
MediaSource::ReadOptions options;
{
Mutex::Autolock autoLock(mLock);
if (mSeeking) {
if (mIsFirstBuffer) {
if (mFirstBuffer != NULL) {
mFirstBuffer->release();
mFirstBuffer = NULL;
}
mIsFirstBuffer = false;
}
options.setSeekTo(mSeekTimeUs);
if (mInputBuffer != NULL) {
mInputBuffer->release();
mInputBuffer = NULL;
}
mSeeking = false;
if (mObserver) {
mObserver->postAudioSeekComplete();
}
}
}
if (mInputBuffer == NULL) {
status_t err;
if (mIsFirstBuffer) {
mInputBuffer = mFirstBuffer;
mFirstBuffer = NULL;
err = mFirstBufferResult;
mIsFirstBuffer = false;
} else {
err = mSource->read(&mInputBuffer, &options);
}
CHECK((err == OK && mInputBuffer != NULL)
|| (err != OK && mInputBuffer == NULL));
Mutex::Autolock autoLock(mLock);
if (err != OK) {
if (mObserver && !mReachedEOS) {
mObserver->postAudioEOS();
}
if (err == INFO_FORMAT_CHANGED) {
sp<MetaData> format = mSource->getFormat();
const char *mime;
bool success = format->findCString(kKeyMIMEType, &mime);
CHECK(success);
CHECK(!strcasecmp(mime, MEDIA_MIMETYPE_AUDIO_RAW));
success = format->findInt32(kKeySampleRate, &mSampleRate);
CHECK(success);
int32_t numChannels;
success = format->findInt32(kKeyChannelCount, &numChannels);
CHECK(success);
mAudioSink->stop();
mAudioSink->close();
status_t err = mAudioSink->open(
mSampleRate, numChannels, AudioSystem::PCM_16_BIT,
DEFAULT_AUDIOSINK_BUFFERCOUNT,
&AudioPlayer::AudioSinkCallback, this);
if (err != OK) {
mSource->stop();
return err;
}
mLatencyUs = (int64_t)mAudioSink->latency() * 1000;
mFrameSize = mAudioSink->frameSize();
mAudioSink->start();
break;
} else if ((err ==INVALID_OPERATION)&&mPaused){
break;
} else {
mReachedEOS = true;
mFinalStatus = err;
break;
}
}
CHECK(mInputBuffer->meta_data()->findInt64(
kKeyTime, &mPositionTimeMediaUs));
mPositionTimeRealUs =
((mNumFramesPlayed + size_done / mFrameSize) * 1000000)
/ mSampleRate;
LOGV("buffer->size() = %d, "
"mPositionTimeMediaUs=%.2f mPositionTimeRealUs=%.2f",
mInputBuffer->range_length(),
mPositionTimeMediaUs / 1E6, mPositionTimeRealUs / 1E6);
}
if (mInputBuffer->range_length() == 0) {
mInputBuffer->release();
mInputBuffer = NULL;
continue;
}
size_t copy = size_remaining;
if (copy > mInputBuffer->range_length()) {
copy = mInputBuffer->range_length();
}
memcpy((char *)data + size_done,
(const char *)mInputBuffer->data() + mInputBuffer->range_offset(),
copy);
mInputBuffer->set_range(mInputBuffer->range_offset() + copy,
mInputBuffer->range_length() - copy);
size_done += copy;
size_remaining -= copy;
}
Mutex::Autolock autoLock(mLock);
mNumFramesPlayed += size_done / mFrameSize;
return size_done;
}
bool OmxDecoder::ReadVideo(VideoFrame *aFrame, int64_t aTimeUs,
bool aKeyframeSkip, bool aDoSeek)
{
if (!mVideoSource.get())
return false;
ReleaseVideoBuffer();
status_t err;
if (aDoSeek) {
{
Mutex::Autolock autoLock(mSeekLock);
ReleaseAllPendingVideoBuffersLocked();
mIsVideoSeeking = true;
}
MediaSource::ReadOptions options;
options.setSeekTo(aTimeUs, MediaSource::ReadOptions::SEEK_PREVIOUS_SYNC);
err = mVideoSource->read(&mVideoBuffer, &options);
{
Mutex::Autolock autoLock(mSeekLock);
mIsVideoSeeking = false;
PostReleaseVideoBuffer(nullptr, FenceHandle());
}
aDoSeek = false;
} else {
err = mVideoSource->read(&mVideoBuffer);
}
aFrame->mSize = 0;
if (err == OK) {
int64_t timeUs;
int32_t unreadable;
int32_t keyFrame;
size_t length = mVideoBuffer->range_length();
if (!mVideoBuffer->meta_data()->findInt64(kKeyTime, &timeUs) ) {
NS_WARNING("OMX decoder did not return frame time");
return false;
}
if (!mVideoBuffer->meta_data()->findInt32(kKeyIsSyncFrame, &keyFrame)) {
keyFrame = 0;
}
if (!mVideoBuffer->meta_data()->findInt32(kKeyIsUnreadable, &unreadable)) {
unreadable = 0;
}
RefPtr<mozilla::layers::TextureClient> textureClient;
if ((mVideoBuffer->graphicBuffer().get())) {
textureClient = mNativeWindow->getTextureClientFromBuffer(mVideoBuffer->graphicBuffer().get());
}
if (textureClient) {
// Manually increment reference count to keep MediaBuffer alive
// during TextureClient is in use.
mVideoBuffer->add_ref();
GrallocTextureClientOGL* grallocClient = static_cast<GrallocTextureClientOGL*>(textureClient.get());
grallocClient->SetMediaBuffer(mVideoBuffer);
// Set recycle callback for TextureClient
textureClient->SetRecycleCallback(OmxDecoder::RecycleCallback, this);
{
Mutex::Autolock autoLock(mPendingVideoBuffersLock);
// Store pending recycle TextureClient.
MOZ_ASSERT(mPendingRecycleTexutreClients.find(textureClient) == mPendingRecycleTexutreClients.end());
mPendingRecycleTexutreClients.insert(textureClient);
}
aFrame->mGraphicBuffer = textureClient;
aFrame->mRotation = mVideoRotation;
aFrame->mTimeUs = timeUs;
aFrame->mKeyFrame = keyFrame;
aFrame->Y.mWidth = mVideoWidth;
aFrame->Y.mHeight = mVideoHeight;
// Release to hold video buffer in OmxDecoder more.
// MediaBuffer's ref count is changed from 2 to 1.
ReleaseVideoBuffer();
} else if (length > 0) {
char *data = static_cast<char *>(mVideoBuffer->data()) + mVideoBuffer->range_offset();
if (unreadable) {
LOG(PR_LOG_DEBUG, "video frame is unreadable");
}
if (!ToVideoFrame(aFrame, timeUs, data, length, keyFrame)) {
return false;
}
}
// Check if this frame is valid or not. If not, skip it.
if ((aKeyframeSkip && timeUs < aTimeUs) || length == 0) {
aFrame->mShouldSkip = true;
}
}
else if (err == INFO_FORMAT_CHANGED) {
// If the format changed, update our cached info.
if (!SetVideoFormat()) {
return false;
} else {
return ReadVideo(aFrame, aTimeUs, aKeyframeSkip, aDoSeek);
}
}
else if (err == ERROR_END_OF_STREAM) {
return false;
}
else if (err == -ETIMEDOUT) {
LOG(PR_LOG_DEBUG, "OmxDecoder::ReadVideo timed out, will retry");
return true;
}
else {
// UNKNOWN_ERROR is sometimes is used to mean "out of memory", but
// regardless, don't keep trying to decode if the decoder doesn't want to.
LOG(PR_LOG_DEBUG, "OmxDecoder::ReadVideo failed, err=%d", err);
return false;
}
return true;
}
status_t Harness::testSeek(
const char *componentName, const char *componentRole) {
bool isEncoder =
!strncmp(componentRole, "audio_encoder.", 14)
|| !strncmp(componentRole, "video_encoder.", 14);
if (isEncoder) {
// Not testing seek behaviour for encoders.
printf(" * Not testing seek functionality for encoders.\n");
return OK;
}
const char *mime = GetMimeFromComponentRole(componentRole);
if (!mime) {
LOGI("Cannot perform seek test with this componentRole (%s)",
componentRole);
return OK;
}
sp<MediaSource> source = CreateSourceForMime(mime);
sp<MediaSource> seekSource = CreateSourceForMime(mime);
if (source == NULL || seekSource == NULL) {
return UNKNOWN_ERROR;
}
CHECK_EQ(seekSource->start(), OK);
sp<MediaSource> codec = OMXCodec::Create(
mOMX, source->getFormat(), false /* createEncoder */,
source, componentName);
CHECK(codec != NULL);
CHECK_EQ(codec->start(), OK);
int64_t durationUs;
CHECK(source->getFormat()->findInt64(kKeyDuration, &durationUs));
LOGI("stream duration is %lld us (%.2f secs)",
durationUs, durationUs / 1E6);
static const int32_t kNumIterations = 5000;
// We are always going to seek beyond EOS in the first iteration (i == 0)
// followed by a linear read for the second iteration (i == 1).
// After that it's all random.
for (int32_t i = 0; i < kNumIterations; ++i) {
int64_t requestedSeekTimeUs;
int64_t actualSeekTimeUs;
MediaSource::ReadOptions options;
double r = uniform_rand();
if ((i == 1) || (i > 0 && r < 0.5)) {
// 50% chance of just continuing to decode from last position.
requestedSeekTimeUs = -1;
LOGI("requesting linear read");
} else {
if (i == 0 || r < 0.55) {
// 5% chance of seeking beyond end of stream.
requestedSeekTimeUs = durationUs;
LOGI("requesting seek beyond EOF");
} else {
requestedSeekTimeUs =
(int64_t)(uniform_rand() * durationUs);
LOGI("requesting seek to %lld us (%.2f secs)",
requestedSeekTimeUs, requestedSeekTimeUs / 1E6);
}
MediaBuffer *buffer = NULL;
options.setSeekTo(
requestedSeekTimeUs, MediaSource::ReadOptions::SEEK_NEXT_SYNC);
if (seekSource->read(&buffer, &options) != OK) {
CHECK_EQ(buffer, NULL);
actualSeekTimeUs = -1;
} else {
CHECK(buffer != NULL);
CHECK(buffer->meta_data()->findInt64(kKeyTime, &actualSeekTimeUs));
CHECK(actualSeekTimeUs >= 0);
buffer->release();
buffer = NULL;
}
LOGI("nearest keyframe is at %lld us (%.2f secs)",
actualSeekTimeUs, actualSeekTimeUs / 1E6);
}
status_t err;
MediaBuffer *buffer;
for (;;) {
err = codec->read(&buffer, &options);
options.clearSeekTo();
if (err == INFO_FORMAT_CHANGED) {
CHECK_EQ(buffer, NULL);
continue;
}
if (err == OK) {
CHECK(buffer != NULL);
if (buffer->range_length() == 0) {
buffer->release();
buffer = NULL;
continue;
}
} else {
CHECK_EQ(buffer, NULL);
}
break;
}
if (requestedSeekTimeUs < 0) {
// Linear read.
if (err != OK) {
CHECK_EQ(buffer, NULL);
} else {
CHECK(buffer != NULL);
buffer->release();
buffer = NULL;
}
} else if (actualSeekTimeUs < 0) {
EXPECT(err != OK,
"We attempted to seek beyond EOS and expected "
"ERROR_END_OF_STREAM to be returned, but instead "
"we got a valid buffer.");
EXPECT(err == ERROR_END_OF_STREAM,
"We attempted to seek beyond EOS and expected "
"ERROR_END_OF_STREAM to be returned, but instead "
"we found some other error.");
CHECK_EQ(err, ERROR_END_OF_STREAM);
CHECK_EQ(buffer, NULL);
} else {
EXPECT(err == OK,
"Expected a valid buffer to be returned from "
"OMXCodec::read.");
CHECK(buffer != NULL);
int64_t bufferTimeUs;
CHECK(buffer->meta_data()->findInt64(kKeyTime, &bufferTimeUs));
if (!CloseEnough(bufferTimeUs, actualSeekTimeUs)) {
printf("\n * Attempted seeking to %lld us (%.2f secs)",
requestedSeekTimeUs, requestedSeekTimeUs / 1E6);
printf("\n * Nearest keyframe is at %lld us (%.2f secs)",
actualSeekTimeUs, actualSeekTimeUs / 1E6);
printf("\n * Returned buffer was at %lld us (%.2f secs)\n\n",
bufferTimeUs, bufferTimeUs / 1E6);
buffer->release();
buffer = NULL;
CHECK_EQ(codec->stop(), OK);
return UNKNOWN_ERROR;
}
buffer->release();
buffer = NULL;
}
}
CHECK_EQ(codec->stop(), OK);
return OK;
}
status_t AudioPlayer::start(bool sourceAlreadyStarted) {
CHECK(!mStarted);
CHECK(mSource != NULL);
status_t err;
if (!sourceAlreadyStarted) {
err = mSource->start();
if (err != OK) {
return err;
}
}
// We allow an optional INFO_FORMAT_CHANGED at the very beginning
// of playback, if there is one, getFormat below will retrieve the
// updated format, if there isn't, we'll stash away the valid buffer
// of data to be used on the first audio callback.
CHECK(mFirstBuffer == NULL);
MediaSource::ReadOptions options;
if (mSeeking) {
options.setSeekTo(mSeekTimeUs);
mSeeking = false;
}
mFirstBufferResult = mSource->read(&mFirstBuffer, &options);
if (mFirstBufferResult == INFO_FORMAT_CHANGED) {
ALOGV("INFO_FORMAT_CHANGED!!!");
CHECK(mFirstBuffer == NULL);
mFirstBufferResult = OK;
mIsFirstBuffer = false;
} else {
mIsFirstBuffer = true;
}
sp<MetaData> format = mSource->getFormat();
const char *mime;
bool success = format->findCString(kKeyMIMEType, &mime);
CHECK(success);
CHECK(!strcasecmp(mime, MEDIA_MIMETYPE_AUDIO_RAW));
success = format->findInt32(kKeySampleRate, &mSampleRate);
CHECK(success);
int32_t numChannels, channelMask;
success = format->findInt32(kKeyChannelCount, &numChannels);
CHECK(success);
if(!format->findInt32(kKeyChannelMask, &channelMask)) {
// log only when there's a risk of ambiguity of channel mask selection
ALOGI_IF(numChannels > 2,
"source format didn't specify channel mask, using (%d) channel order", numChannels);
channelMask = CHANNEL_MASK_USE_CHANNEL_ORDER;
}
if (mAudioSink.get() != NULL) {
status_t err = mAudioSink->open(
mSampleRate, numChannels, channelMask, AUDIO_FORMAT_PCM_16_BIT,
DEFAULT_AUDIOSINK_BUFFERCOUNT,
&AudioPlayer::AudioSinkCallback,
this,
(mAllowDeepBuffering ?
AUDIO_OUTPUT_FLAG_DEEP_BUFFER :
AUDIO_OUTPUT_FLAG_NONE));
if (err != OK) {
if (mFirstBuffer != NULL) {
mFirstBuffer->release();
mFirstBuffer = NULL;
}
if (!sourceAlreadyStarted) {
mSource->stop();
}
return err;
}
mLatencyUs = (int64_t)mAudioSink->latency() * 1000;
mFrameSize = mAudioSink->frameSize();
mAudioSink->start();
} else {
// playing to an AudioTrack, set up mask if necessary
audio_channel_mask_t audioMask = channelMask == CHANNEL_MASK_USE_CHANNEL_ORDER ?
audio_channel_out_mask_from_count(numChannels) : channelMask;
if (0 == audioMask) {
return BAD_VALUE;
}
mAudioTrack = new AudioTrack(
AUDIO_STREAM_MUSIC, mSampleRate, AUDIO_FORMAT_PCM_16_BIT, audioMask,
0, AUDIO_OUTPUT_FLAG_NONE, &AudioCallback, this, 0);
if ((err = mAudioTrack->initCheck()) != OK) {
delete mAudioTrack;
mAudioTrack = NULL;
if (mFirstBuffer != NULL) {
mFirstBuffer->release();
mFirstBuffer = NULL;
}
if (!sourceAlreadyStarted) {
mSource->stop();
}
return err;
}
mLatencyUs = (int64_t)mAudioTrack->latency() * 1000;
mFrameSize = mAudioTrack->frameSize();
mAudioTrack->start();
}
mStarted = true;
mPinnedTimeUs = -1ll;
return OK;
}
static VideoFrame *extractVideoFrameWithCodecFlags(
OMXClient *client,
const sp<MetaData> &trackMeta,
const sp<MediaSource> &source, uint32_t flags) {
sp<MediaSource> decoder =
OMXCodec::Create(
client->interface(), source->getFormat(), false, source,
NULL, flags | OMXCodec::kClientNeedsFramebuffer);
if (decoder.get() == NULL) {
LOGV("unable to instantiate video decoder.");
return NULL;
}
status_t err = decoder->start();
if (err != OK) {
LOGW("OMXCodec::start returned error %d (0x%08x)\n", err, err);
return NULL;
}
// Read one output buffer, ignore format change notifications
// and spurious empty buffers.
MediaSource::ReadOptions options;
int64_t thumbNailTime;
if (trackMeta->findInt64(kKeyThumbnailTime, &thumbNailTime)) {
options.setSeekTo(thumbNailTime);
} else {
thumbNailTime = -1;
}
MediaBuffer *buffer = NULL;
do {
if (buffer != NULL) {
buffer->release();
buffer = NULL;
}
err = decoder->read(&buffer, &options);
options.clearSeekTo();
} while (err == INFO_FORMAT_CHANGED
|| (buffer != NULL && buffer->range_length() == 0));
if (err != OK) {
CHECK_EQ(buffer, NULL);
LOGV("decoding frame failed.");
decoder->stop();
return NULL;
}
LOGV("successfully decoded video frame.");
int32_t unreadable;
if (buffer->meta_data()->findInt32(kKeyIsUnreadable, &unreadable)
&& unreadable != 0) {
LOGV("video frame is unreadable, decoder does not give us access "
"to the video data.");
buffer->release();
buffer = NULL;
decoder->stop();
return NULL;
}
int64_t timeUs;
CHECK(buffer->meta_data()->findInt64(kKeyTime, &timeUs));
if (thumbNailTime >= 0) {
if (timeUs != thumbNailTime) {
const char *mime;
CHECK(trackMeta->findCString(kKeyMIMEType, &mime));
LOGV("thumbNailTime = %lld us, timeUs = %lld us, mime = %s",
thumbNailTime, timeUs, mime);
}
}
sp<MetaData> meta = decoder->getFormat();
int32_t width, height;
CHECK(meta->findInt32(kKeyWidth, &width));
CHECK(meta->findInt32(kKeyHeight, &height));
int32_t rotationAngle;
if (!trackMeta->findInt32(kKeyRotation, &rotationAngle)) {
rotationAngle = 0; // By default, no rotation
}
VideoFrame *frame = new VideoFrame;
frame->mWidth = width;
frame->mHeight = height;
frame->mDisplayWidth = width;
frame->mDisplayHeight = height;
frame->mSize = width * height * 2;
frame->mData = new uint8_t[frame->mSize];
frame->mRotationAngle = rotationAngle;
int32_t srcFormat;
CHECK(meta->findInt32(kKeyColorFormat, &srcFormat));
ColorConverter converter(
(OMX_COLOR_FORMATTYPE)srcFormat, OMX_COLOR_Format16bitRGB565);
CHECK(converter.isValid());
converter.convert(
width, height,
(const uint8_t *)buffer->data() + buffer->range_offset(),
0,
frame->mData, width * 2);
buffer->release();
buffer = NULL;
decoder->stop();
return frame;
}
static VideoFrame *extractVideoFrameWithCodecFlags(
OMXClient *client,
const sp<MetaData> &trackMeta,
const sp<MediaSource> &source,
uint32_t flags,
int64_t frameTimeUs,
int seekMode) {
sp<MetaData> format = source->getFormat();
#ifndef MTK_HARDWARE
// XXX:
// Once all vendors support OMX_COLOR_FormatYUV420Planar, we can
// remove this check and always set the decoder output color format
// skip this check for software decoders
#ifndef QCOM_HARDWARE
if (isYUV420PlanarSupported(client, trackMeta)) {
format->setInt32(kKeyColorFormat, OMX_COLOR_FormatYUV420Planar);
#else
if (!(flags & OMXCodec::kSoftwareCodecsOnly)) {
if (isYUV420PlanarSupported(client, trackMeta)) {
format->setInt32(kKeyColorFormat, OMX_COLOR_FormatYUV420Planar);
}
#endif
}
#endif
sp<MediaSource> decoder =
OMXCodec::Create(
client->interface(), format, false, source,
NULL, flags | OMXCodec::kClientNeedsFramebuffer);
if (decoder.get() == NULL) {
ALOGV("unable to instantiate video decoder.");
return NULL;
}
status_t err = decoder->start();
if (err != OK) {
ALOGW("OMXCodec::start returned error %d (0x%08x)\n", err, err);
return NULL;
}
// Read one output buffer, ignore format change notifications
// and spurious empty buffers.
MediaSource::ReadOptions options;
if (seekMode < MediaSource::ReadOptions::SEEK_PREVIOUS_SYNC ||
seekMode > MediaSource::ReadOptions::SEEK_CLOSEST) {
ALOGE("Unknown seek mode: %d", seekMode);
return NULL;
}
MediaSource::ReadOptions::SeekMode mode =
static_cast<MediaSource::ReadOptions::SeekMode>(seekMode);
int64_t thumbNailTime;
if (frameTimeUs < 0) {
if (!trackMeta->findInt64(kKeyThumbnailTime, &thumbNailTime)
|| thumbNailTime < 0) {
thumbNailTime = 0;
}
options.setSeekTo(thumbNailTime, mode);
} else {
thumbNailTime = -1;
options.setSeekTo(frameTimeUs, mode);
}
MediaBuffer *buffer = NULL;
do {
if (buffer != NULL) {
buffer->release();
buffer = NULL;
}
err = decoder->read(&buffer, &options);
options.clearSeekTo();
} while (err == INFO_FORMAT_CHANGED
|| (buffer != NULL && buffer->range_length() == 0));
if (err != OK) {
CHECK(buffer == NULL);
ALOGV("decoding frame failed.");
decoder->stop();
return NULL;
}
ALOGV("successfully decoded video frame.");
int32_t unreadable;
if (buffer->meta_data()->findInt32(kKeyIsUnreadable, &unreadable)
&& unreadable != 0) {
ALOGV("video frame is unreadable, decoder does not give us access "
"to the video data.");
buffer->release();
buffer = NULL;
decoder->stop();
return NULL;
}
int64_t timeUs;
CHECK(buffer->meta_data()->findInt64(kKeyTime, &timeUs));
if (thumbNailTime >= 0) {
if (timeUs != thumbNailTime) {
const char *mime;
CHECK(trackMeta->findCString(kKeyMIMEType, &mime));
ALOGV("thumbNailTime = %lld us, timeUs = %lld us, mime = %s",
thumbNailTime, timeUs, mime);
}
}
sp<MetaData> meta = decoder->getFormat();
int32_t width, height;
CHECK(meta->findInt32(kKeyWidth, &width));
CHECK(meta->findInt32(kKeyHeight, &height));
int32_t crop_left, crop_top, crop_right, crop_bottom;
if (!meta->findRect(
kKeyCropRect,
&crop_left, &crop_top, &crop_right, &crop_bottom)) {
crop_left = crop_top = 0;
crop_right = width - 1;
crop_bottom = height - 1;
}
int32_t rotationAngle;
if (!trackMeta->findInt32(kKeyRotation, &rotationAngle)) {
rotationAngle = 0; // By default, no rotation
}
VideoFrame *frame = new VideoFrame;
frame->mWidth = crop_right - crop_left + 1;
frame->mHeight = crop_bottom - crop_top + 1;
frame->mDisplayWidth = frame->mWidth;
frame->mDisplayHeight = frame->mHeight;
frame->mSize = frame->mWidth * frame->mHeight * 2;
frame->mData = new uint8_t[frame->mSize];
frame->mRotationAngle = rotationAngle;
int32_t displayWidth, displayHeight;
if (meta->findInt32(kKeyDisplayWidth, &displayWidth)) {
frame->mDisplayWidth = displayWidth;
}
if (meta->findInt32(kKeyDisplayHeight, &displayHeight)) {
frame->mDisplayHeight = displayHeight;
}
int32_t srcFormat;
CHECK(meta->findInt32(kKeyColorFormat, &srcFormat));
#ifdef MTK_HARDWARE
{
int32_t Stridewidth,SliceHeight;
CHECK(meta->findInt32(kKeyStride, &Stridewidth));
CHECK(meta->findInt32(kKeySliceHeight, &SliceHeight));
ALOGD("kKeyWidth=%d,kKeyHeight=%d",width,height);
ALOGD("Stridewidth=%d,SliceHeight=%d",Stridewidth,SliceHeight);
width=Stridewidth;
height=SliceHeight;
}
#endif
ColorConverter converter((OMX_COLOR_FORMATTYPE)srcFormat, OMX_COLOR_Format16bitRGB565);
if (converter.isValid()) {
err = converter.convert(
(const uint8_t *)buffer->data() + buffer->range_offset(),
width, height,
crop_left, crop_top, crop_right, crop_bottom,
frame->mData,
frame->mWidth,
frame->mHeight,
0, 0, frame->mWidth - 1, frame->mHeight - 1);
} else {
ALOGE("Unable to instantiate color conversion from format 0x%08x to "
"RGB565",
srcFormat);
err = ERROR_UNSUPPORTED;
}
buffer->release();
buffer = NULL;
decoder->stop();
if (err != OK) {
ALOGE("Colorconverter failed to convert frame.");
delete frame;
frame = NULL;
}
return frame;
}
VideoFrame *StagefrightMetadataRetriever::getFrameAtTime(
int64_t timeUs, int option) {
ALOGV("getFrameAtTime: %lld us option: %d", timeUs, option);
if (mExtractor.get() == NULL) {
ALOGV("no extractor.");
return NULL;
}
sp<MetaData> fileMeta = mExtractor->getMetaData();
if (fileMeta == NULL) {
ALOGV("extractor doesn't publish metadata, failed to initialize?");
return NULL;
}
int32_t drm = 0;
if (fileMeta->findInt32(kKeyIsDRM, &drm) && drm != 0) {
ALOGE("frame grab not allowed.");
return NULL;
}
size_t n = mExtractor->countTracks();
size_t i;
for (i = 0; i < n; ++i) {
sp<MetaData> meta = mExtractor->getTrackMetaData(i);
const char *mime;
CHECK(meta->findCString(kKeyMIMEType, &mime));
if (!strncasecmp(mime, "video/", 6)) {
break;
}
}
if (i == n) {
ALOGV("no video track found.");
return NULL;
}
sp<MetaData> trackMeta = mExtractor->getTrackMetaData(
i, MediaExtractor::kIncludeExtensiveMetaData);
sp<MediaSource> source = mExtractor->getTrack(i);
if (source.get() == NULL) {
ALOGV("unable to instantiate video track.");
return NULL;
}
const void *data;
uint32_t type;
size_t dataSize;
if (fileMeta->findData(kKeyAlbumArt, &type, &data, &dataSize)
&& mAlbumArt == NULL) {
mAlbumArt = new MediaAlbumArt;
mAlbumArt->mSize = dataSize;
mAlbumArt->mData = new uint8_t[dataSize];
memcpy(mAlbumArt->mData, data, dataSize);
}
VideoFrame *frame =
extractVideoFrameWithCodecFlags(
#ifndef QCOM_HARDWARE
&mClient, trackMeta, source, OMXCodec::kPreferSoftwareCodecs,
#else
&mClient, trackMeta, source, OMXCodec::kSoftwareCodecsOnly,
#endif
timeUs, option);
if (frame == NULL) {
ALOGV("Software decoder failed to extract thumbnail, "
"trying hardware decoder.");
frame = extractVideoFrameWithCodecFlags(&mClient, trackMeta, source, 0,
timeUs, option);
}
return frame;
}
static VideoFrame *extractVideoFrameWithCodecFlags(
OMXClient *client,
const sp<MetaData> &trackMeta,
const sp<MediaSource> &source,
uint32_t flags,
int64_t frameTimeUs,
int seekMode) {
sp<MetaData> format = source->getFormat();
// XXX:
// Once all vendors support OMX_COLOR_FormatYUV420Planar, we can
// remove this check and always set the decoder output color format
if (isYUV420PlanarSupported(client, trackMeta)) {
format->setInt32(kKeyColorFormat, OMX_COLOR_FormatYUV420Planar);
}
sp<MediaSource> decoder =
OMXCodec::Create(
client->interface(), format, false, source,
NULL, flags | OMXCodec::kClientNeedsFramebuffer);
if (decoder.get() == NULL) {
ALOGV("unable to instantiate video decoder.");
return NULL;
}
status_t err = decoder->start();
if (err != OK) {
ALOGW("OMXCodec::start returned error %d (0x%08x)\n", err, err);
return NULL;
}
// Read one output buffer, ignore format change notifications
// and spurious empty buffers.
MediaSource::ReadOptions options;
if (seekMode < MediaSource::ReadOptions::SEEK_PREVIOUS_SYNC ||
seekMode > MediaSource::ReadOptions::SEEK_CLOSEST) {
ALOGE("Unknown seek mode: %d", seekMode);
return NULL;
}
MediaSource::ReadOptions::SeekMode mode =
static_cast<MediaSource::ReadOptions::SeekMode>(seekMode);
int64_t thumbNailTime;
if (frameTimeUs < 0) {
if (!trackMeta->findInt64(kKeyThumbnailTime, &thumbNailTime)
|| thumbNailTime < 0) {
thumbNailTime = 0;
}
options.setSeekTo(thumbNailTime, mode);
} else {
thumbNailTime = -1;
options.setSeekTo(frameTimeUs, mode);
}
MediaBuffer *buffer = NULL;
do {
if (buffer != NULL) {
buffer->release();
buffer = NULL;
}
err = decoder->read(&buffer, &options);
options.clearSeekTo();
} while (err == INFO_FORMAT_CHANGED
|| (buffer != NULL && buffer->range_length() == 0));
if (err != OK) {
CHECK(buffer == NULL);
ALOGV("decoding frame failed.");
decoder->stop();
return NULL;
}
ALOGV("successfully decoded video frame.");
int32_t unreadable;
if (buffer->meta_data()->findInt32(kKeyIsUnreadable, &unreadable)
&& unreadable != 0) {
ALOGV("video frame is unreadable, decoder does not give us access "
"to the video data.");
buffer->release();
buffer = NULL;
decoder->stop();
return NULL;
}
int64_t timeUs;
CHECK(buffer->meta_data()->findInt64(kKeyTime, &timeUs));
if (thumbNailTime >= 0) {
if (timeUs != thumbNailTime) {
const char *mime;
CHECK(trackMeta->findCString(kKeyMIMEType, &mime));
ALOGV("thumbNailTime = %" PRId64 " us, timeUs = %" PRId64 " us, mime = %s",
thumbNailTime, timeUs, mime);
}
}
sp<MetaData> meta = decoder->getFormat();
int32_t width, height;
CHECK(meta->findInt32(kKeyWidth, &width));
CHECK(meta->findInt32(kKeyHeight, &height));
int32_t crop_left, crop_top, crop_right, crop_bottom;
if (!meta->findRect(
kKeyCropRect,
&crop_left, &crop_top, &crop_right, &crop_bottom)) {
crop_left = crop_top = 0;
crop_right = width - 1;
crop_bottom = height - 1;
}
int32_t rotationAngle;
if (!trackMeta->findInt32(kKeyRotation, &rotationAngle)) {
rotationAngle = 0; // By default, no rotation
}
VideoFrame *frame = new VideoFrame;
frame->mWidth = crop_right - crop_left + 1;
frame->mHeight = crop_bottom - crop_top + 1;
frame->mDisplayWidth = frame->mWidth;
frame->mDisplayHeight = frame->mHeight;
frame->mSize = frame->mWidth * frame->mHeight * 2;
frame->mData = new uint8_t[frame->mSize];
frame->mRotationAngle = rotationAngle;
int32_t displayWidth, displayHeight;
if (meta->findInt32(kKeyDisplayWidth, &displayWidth)) {
frame->mDisplayWidth = displayWidth;
}
if (meta->findInt32(kKeyDisplayHeight, &displayHeight)) {
frame->mDisplayHeight = displayHeight;
}
int32_t srcFormat;
CHECK(meta->findInt32(kKeyColorFormat, &srcFormat));
ColorConverter converter(
(OMX_COLOR_FORMATTYPE)srcFormat, OMX_COLOR_Format16bitRGB565);
if (converter.isValid()) {
err = converter.convert(
(const uint8_t *)buffer->data() + buffer->range_offset(),
width, height,
crop_left, crop_top, crop_right, crop_bottom,
frame->mData,
frame->mWidth,
frame->mHeight,
0, 0, frame->mWidth - 1, frame->mHeight - 1);
} else {
ALOGE("Unable to instantiate color conversion from format 0x%08x to "
"RGB565",
srcFormat);
err = ERROR_UNSUPPORTED;
}
buffer->release();
buffer = NULL;
decoder->stop();
if (err != OK) {
ALOGE("Colorconverter failed to convert frame.");
delete frame;
frame = NULL;
}
return frame;
}