void
CameraPreviewMediaStream::SetCurrentFrame(const gfx::IntSize& aIntrinsicSize, Image* aImage)
{
{
MutexAutoLock lock(mMutex);
if (mInvalidatePending > 0) {
if (mRateLimit || mInvalidatePending > MAX_INVALIDATE_PENDING) {
++mDiscardedFrames;
DOM_CAMERA_LOGW("Discard preview frame %d, %d invalidation(s) pending",
mDiscardedFrames, mInvalidatePending);
return;
}
DOM_CAMERA_LOGI("Update preview frame, %d invalidation(s) pending",
mInvalidatePending);
}
mDiscardedFrames = 0;
TimeStamp now = TimeStamp::Now();
for (nsTArray<RefPtr<VideoFrameContainer> >::size_type i = 0; i < mVideoOutputs.Length(); ++i) {
VideoFrameContainer* output = mVideoOutputs[i];
output->SetCurrentFrame(aIntrinsicSize, aImage, now);
}
++mInvalidatePending;
}
NS_DispatchToMainThread(NewRunnableMethod(this, &CameraPreviewMediaStream::Invalidate));
}
nsRefPtr<MediaDecoderReader::SeekPromise>
MediaOmxReader::Seek(int64_t aTarget, int64_t aEndTime)
{
NS_ASSERTION(mDecoder->OnDecodeThread(), "Should be on decode thread.");
EnsureActive();
VideoFrameContainer* container = mDecoder->GetVideoFrameContainer();
if (container && container->GetImageContainer()) {
container->GetImageContainer()->ClearAllImagesExceptFront();
}
if (mHasAudio && mHasVideo) {
// The OMXDecoder seeks/demuxes audio and video streams separately. So if
// we seek both audio and video to aTarget, the audio stream can typically
// seek closer to the seek target, since typically every audio block is
// a sync point, whereas for video there are only keyframes once every few
// seconds. So if we have both audio and video, we must seek the video
// stream to the preceeding keyframe first, get the stream time, and then
// seek the audio stream to match the video stream's time. Otherwise, the
// audio and video streams won't be in sync after the seek.
mVideoSeekTimeUs = aTarget;
const VideoData* v = DecodeToFirstVideoData();
mAudioSeekTimeUs = v ? v->mTime : aTarget;
} else {
mAudioSeekTimeUs = mVideoSeekTimeUs = aTarget;
}
return SeekPromise::CreateAndResolve(mAudioSeekTimeUs, __func__);
}
nsresult MediaOmxReader::ReadMetadata(VideoInfo* aInfo,
MetadataTags** aTags)
{
NS_ASSERTION(mDecoder->OnDecodeThread(), "Should be on decode thread.");
*aTags = nullptr;
if (!mOmxDecoder.get()) {
mOmxDecoder = new OmxDecoder(mDecoder->GetResource(), mDecoder);
if (!mOmxDecoder->Init()) {
return NS_ERROR_FAILURE;
}
}
// Set the total duration (the max of the audio and video track).
int64_t durationUs;
mOmxDecoder->GetDuration(&durationUs);
if (durationUs) {
ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor());
mDecoder->SetMediaDuration(durationUs);
}
if (mOmxDecoder->HasVideo()) {
int32_t width, height;
mOmxDecoder->GetVideoParameters(&width, &height);
nsIntRect pictureRect(0, 0, width, height);
// Validate the container-reported frame and pictureRect sizes. This ensures
// that our video frame creation code doesn't overflow.
nsIntSize displaySize(width, height);
nsIntSize frameSize(width, height);
if (!VideoInfo::ValidateVideoRegion(frameSize, pictureRect, displaySize)) {
return NS_ERROR_FAILURE;
}
// Video track's frame sizes will not overflow. Activate the video track.
mHasVideo = mInfo.mHasVideo = true;
mInfo.mDisplay = displaySize;
mPicture = pictureRect;
mInitialFrame = frameSize;
VideoFrameContainer* container = mDecoder->GetVideoFrameContainer();
if (container) {
container->SetCurrentFrame(gfxIntSize(displaySize.width, displaySize.height),
nullptr,
mozilla::TimeStamp::Now());
}
}
if (mOmxDecoder->HasAudio()) {
int32_t numChannels, sampleRate;
mOmxDecoder->GetAudioParameters(&numChannels, &sampleRate);
mHasAudio = mInfo.mHasAudio = true;
mInfo.mAudioChannels = numChannels;
mInfo.mAudioRate = sampleRate;
}
*aInfo = mInfo;
return NS_OK;
}
nsresult AndroidMediaReader::ReadMetadata(MediaInfo* aInfo,
MetadataTags** aTags)
{
MOZ_ASSERT(OnTaskQueue());
if (!mPlugin) {
mPlugin = GetAndroidMediaPluginHost()->CreateDecoder(mDecoder->GetResource(), mType);
if (!mPlugin) {
return NS_ERROR_FAILURE;
}
}
// Set the total duration (the max of the audio and video track).
int64_t durationUs;
mPlugin->GetDuration(mPlugin, &durationUs);
if (durationUs) {
ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor());
mDecoder->SetMediaDuration(durationUs);
}
if (mPlugin->HasVideo(mPlugin)) {
int32_t width, height;
mPlugin->GetVideoParameters(mPlugin, &width, &height);
nsIntRect pictureRect(0, 0, width, height);
// Validate the container-reported frame and pictureRect sizes. This ensures
// that our video frame creation code doesn't overflow.
nsIntSize displaySize(width, height);
nsIntSize frameSize(width, height);
if (!IsValidVideoRegion(frameSize, pictureRect, displaySize)) {
return NS_ERROR_FAILURE;
}
// Video track's frame sizes will not overflow. Activate the video track.
mHasVideo = mInfo.mVideo.mHasVideo = true;
mInfo.mVideo.mDisplay = displaySize;
mPicture = pictureRect;
mInitialFrame = frameSize;
VideoFrameContainer* container = mDecoder->GetVideoFrameContainer();
if (container) {
container->SetCurrentFrame(gfxIntSize(displaySize.width, displaySize.height),
nullptr,
mozilla::TimeStamp::Now());
}
}
if (mPlugin->HasAudio(mPlugin)) {
int32_t numChannels, sampleRate;
mPlugin->GetAudioParameters(mPlugin, &numChannels, &sampleRate);
mHasAudio = mInfo.mAudio.mHasAudio = true;
mInfo.mAudio.mChannels = numChannels;
mInfo.mAudio.mRate = sampleRate;
}
*aInfo = mInfo;
*aTags = nullptr;
return NS_OK;
}
void
CameraPreviewMediaStream::Invalidate()
{
MutexAutoLock lock(mMutex);
--mInvalidatePending;
for (nsTArray<RefPtr<VideoFrameContainer> >::size_type i = 0; i < mVideoOutputs.Length(); ++i) {
VideoFrameContainer* output = mVideoOutputs[i];
output->Invalidate();
}
}
// Resets all state related to decoding, emptying all buffers etc.
nsresult nsMediaOmxReader::ResetDecode()
{
nsBuiltinDecoderReader::ResetDecode();
VideoFrameContainer* container = mDecoder->GetVideoFrameContainer();
if (container) {
container->ClearCurrentFrame();
}
return NS_OK;
}
double HTMLVideoElement::MozFrameDelay()
{
MOZ_ASSERT(NS_IsMainThread(), "Should be on main thread.");
VideoFrameContainer* container = GetVideoFrameContainer();
// Hide negative delays. Frame timing tweaks in the compositor (e.g.
// adding a bias value to prevent multiple dropped/duped frames when
// frame times are aligned with composition times) may produce apparent
// negative delay, but we shouldn't report that.
return container ? std::max(0.0, container->GetFrameDelay()) : 0.0;
}
void
CameraPreviewMediaStream::ClearCurrentFrame()
{
MutexAutoLock lock(mMutex);
for (nsTArray<RefPtr<VideoFrameContainer> >::size_type i = 0; i < mVideoOutputs.Length(); ++i) {
VideoFrameContainer* output = mVideoOutputs[i];
output->ClearCurrentFrame();
NS_DispatchToMainThread(NewRunnableMethod(output, &VideoFrameContainer::Invalidate));
}
}
// Resets all state related to decoding, emptying all buffers etc.
nsresult MediaOmxReader::ResetDecode()
{
MediaDecoderReader::ResetDecode();
VideoFrameContainer* container = mDecoder->GetVideoFrameContainer();
if (container) {
container->ClearCurrentFrame();
}
mOmxDecoder.clear();
return NS_OK;
}
void MP4Reader::ReleaseMediaResources()
{
// Before freeing a video codec, all video buffers needed to be released
// even from graphics pipeline.
VideoFrameContainer* container = mDecoder->GetVideoFrameContainer();
if (container) {
container->ClearCurrentFrame();
}
if (mVideo.mDecoder) {
mVideo.mDecoder->ReleaseMediaResources();
}
}
void HTMLVideoElement::Invalidate(bool aImageSizeChanged,
Maybe<nsIntSize>& aNewIntrinsicSize,
bool aForceInvalidate) {
HTMLMediaElement::Invalidate(aImageSizeChanged, aNewIntrinsicSize,
aForceInvalidate);
if (mVisualCloneTarget) {
VideoFrameContainer* container =
mVisualCloneTarget->GetVideoFrameContainer();
if (container) {
container->Invalidate();
}
}
}
void
CameraPreviewMediaStream::SetCurrentFrame(const gfxIntSize& aIntrinsicSize, Image* aImage)
{
MutexAutoLock lock(mMutex);
TimeStamp now = TimeStamp::Now();
for (uint32_t i = 0; i < mVideoOutputs.Length(); ++i) {
VideoFrameContainer* output = mVideoOutputs[i];
output->SetCurrentFrame(aIntrinsicSize, aImage, now);
nsCOMPtr<nsIRunnable> event =
NS_NewRunnableMethod(output, &VideoFrameContainer::Invalidate);
NS_DispatchToMainThread(event, NS_DISPATCH_NORMAL);
}
}
nsresult MediaOmxReader::Seek(int64_t aTarget, int64_t aStartTime, int64_t aEndTime, int64_t aCurrentTime)
{
NS_ASSERTION(mDecoder->OnDecodeThread(), "Should be on decode thread.");
ResetDecode();
VideoFrameContainer* container = mDecoder->GetVideoFrameContainer();
if (container && container->GetImageContainer()) {
container->GetImageContainer()->ClearAllImagesExceptFront();
}
mAudioSeekTimeUs = mVideoSeekTimeUs = aTarget;
return DecodeToTarget(aTarget);
}
void MediaOmxReader::ReleaseMediaResources()
{
mMediaResourceRequest.DisconnectIfExists();
mMetadataPromise.RejectIfExists(ReadMetadataFailureReason::METADATA_ERROR, __func__);
ResetDecode();
// Before freeing a video codec, all video buffers needed to be released
// even from graphics pipeline.
VideoFrameContainer* container = mDecoder->GetVideoFrameContainer();
if (container) {
container->ClearCurrentFrame();
}
if (mOmxDecoder.get()) {
mOmxDecoder->ReleaseMediaResources();
}
}
// Resets all state related to decoding, emptying all buffers etc.
nsresult MediaOmxReader::ResetDecode()
{
MediaDecoderReader::ResetDecode();
VideoFrameContainer* container = mDecoder->GetVideoFrameContainer();
if (container) {
container->ClearCurrentFrame();
}
if (mLastVideoFrame) {
delete mLastVideoFrame;
mLastVideoFrame = nullptr;
}
if (mOmxDecoder) {
delete mOmxDecoder;
mOmxDecoder = nullptr;
}
return NS_OK;
}
void MediaOmxReader::HandleResourceAllocated()
{
EnsureActive();
// After resources are available, set the metadata.
if (!mOmxDecoder->EnsureMetadata()) {
mMetadataPromise.Reject(ReadMetadataFailureReason::METADATA_ERROR, __func__);
return;
}
bool isMP3 = mDecoder->GetResource()->GetContentType().EqualsASCII(AUDIO_MP3);
if (isMP3 && mMP3FrameParser.IsMP3()) {
// Check if the MP3 frame parser found a duration.
mLastParserDuration = mMP3FrameParser.GetDuration();
}
if (mLastParserDuration >= 0) {
// Prefer the parser duration if we have it.
mInfo.mMetadataDuration = Some(TimeUnit::FromMicroseconds(mLastParserDuration));
} else {
// MP3 parser failed to find a duration.
// Set the total duration (the max of the audio and video track).
int64_t durationUs;
mOmxDecoder->GetDuration(&durationUs);
if (durationUs) {
mInfo.mMetadataDuration = Some(TimeUnit::FromMicroseconds(durationUs));
}
}
if (mOmxDecoder->HasVideo()) {
int32_t displayWidth, displayHeight, width, height;
mOmxDecoder->GetVideoParameters(&displayWidth, &displayHeight,
&width, &height);
nsIntRect pictureRect(0, 0, width, height);
// Validate the container-reported frame and pictureRect sizes. This ensures
// that our video frame creation code doesn't overflow.
nsIntSize displaySize(displayWidth, displayHeight);
nsIntSize frameSize(width, height);
if (!IsValidVideoRegion(frameSize, pictureRect, displaySize)) {
mMetadataPromise.Reject(ReadMetadataFailureReason::METADATA_ERROR, __func__);
return;
}
// Video track's frame sizes will not overflow. Activate the video track.
mHasVideo = true;
mInfo.mVideo.mDisplay = displaySize;
mPicture = pictureRect;
mInitialFrame = frameSize;
VideoFrameContainer* container = mDecoder->GetVideoFrameContainer();
if (container) {
container->ClearCurrentFrame(gfxIntSize(displaySize.width, displaySize.height));
}
}
if (mOmxDecoder->HasAudio()) {
int32_t numChannels, sampleRate;
mOmxDecoder->GetAudioParameters(&numChannels, &sampleRate);
mHasAudio = true;
mInfo.mAudio.mChannels = numChannels;
mInfo.mAudio.mRate = sampleRate;
}
nsRefPtr<MetadataHolder> metadata = new MetadataHolder();
metadata->mInfo = mInfo;
metadata->mTags = nullptr;
#ifdef MOZ_AUDIO_OFFLOAD
CheckAudioOffload();
#endif
mMetadataPromise.Resolve(metadata, __func__);
}
nsresult MediaOmxReader::ReadMetadata(MediaInfo* aInfo,
MetadataTags** aTags)
{
NS_ASSERTION(mDecoder->OnDecodeThread(), "Should be on decode thread.");
EnsureActive();
*aTags = nullptr;
// Initialize the internal OMX Decoder.
nsresult rv = InitOmxDecoder();
if (NS_FAILED(rv)) {
return rv;
}
if (!mOmxDecoder->TryLoad()) {
return NS_ERROR_FAILURE;
}
#ifdef MOZ_AUDIO_OFFLOAD
CheckAudioOffload();
#endif
if (IsWaitingMediaResources()) {
return NS_OK;
}
// Set the total duration (the max of the audio and video track).
int64_t durationUs;
mOmxDecoder->GetDuration(&durationUs);
if (durationUs) {
ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor());
mDecoder->SetMediaDuration(durationUs);
}
if (mOmxDecoder->HasVideo()) {
int32_t displayWidth, displayHeight, width, height;
mOmxDecoder->GetVideoParameters(&displayWidth, &displayHeight,
&width, &height);
nsIntRect pictureRect(0, 0, width, height);
// Validate the container-reported frame and pictureRect sizes. This ensures
// that our video frame creation code doesn't overflow.
nsIntSize displaySize(displayWidth, displayHeight);
nsIntSize frameSize(width, height);
if (!IsValidVideoRegion(frameSize, pictureRect, displaySize)) {
return NS_ERROR_FAILURE;
}
// Video track's frame sizes will not overflow. Activate the video track.
mHasVideo = mInfo.mVideo.mHasVideo = true;
mInfo.mVideo.mDisplay = displaySize;
mPicture = pictureRect;
mInitialFrame = frameSize;
VideoFrameContainer* container = mDecoder->GetVideoFrameContainer();
if (container) {
container->SetCurrentFrame(gfxIntSize(displaySize.width, displaySize.height),
nullptr,
mozilla::TimeStamp::Now());
}
}
if (mOmxDecoder->HasAudio()) {
int32_t numChannels, sampleRate;
mOmxDecoder->GetAudioParameters(&numChannels, &sampleRate);
mHasAudio = mInfo.mAudio.mHasAudio = true;
mInfo.mAudio.mChannels = numChannels;
mInfo.mAudio.mRate = sampleRate;
}
*aInfo = mInfo;
return NS_OK;
}
nsresult MediaOmxReader::ReadMetadata(MediaInfo* aInfo,
MetadataTags** aTags)
{
NS_ASSERTION(mDecoder->OnDecodeThread(), "Should be on decode thread.");
EnsureActive();
*aTags = nullptr;
// Initialize the internal OMX Decoder.
nsresult rv = InitOmxDecoder();
if (NS_FAILED(rv)) {
return rv;
}
bool isMP3 = mDecoder->GetResource()->GetContentType().EqualsASCII(AUDIO_MP3);
if (isMP3) {
// When read sdcard's file on b2g platform at constructor,
// the mDecoder->GetResource()->GetLength() would return -1.
// Delay set the total duration on this function.
mMP3FrameParser.SetLength(mDecoder->GetResource()->GetLength());
ProcessCachedData(0, true);
}
if (!mOmxDecoder->AllocateMediaResources()) {
return NS_ERROR_FAILURE;
}
// Bug 1050667, both MediaDecoderStateMachine and MediaOmxReader
// relies on IsWaitingMediaResources() function. And the waiting state will be
// changed by binder thread, so we store the waiting state in a cache value to
// make them in consistent state.
UpdateIsWaitingMediaResources();
if (IsWaitingMediaResources()) {
return NS_OK;
}
// After resources are available, set the metadata.
if (!mOmxDecoder->EnsureMetadata()) {
return NS_ERROR_FAILURE;
}
if (isMP3 && mMP3FrameParser.IsMP3()) {
int64_t duration = mMP3FrameParser.GetDuration();
// The MP3FrameParser may reported a duration;
// return -1 if no frame has been parsed.
if (duration >= 0) {
ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor());
mUseParserDuration = true;
mLastParserDuration = duration;
mDecoder->SetMediaDuration(mLastParserDuration);
}
} else {
// Set the total duration (the max of the audio and video track).
int64_t durationUs;
mOmxDecoder->GetDuration(&durationUs);
if (durationUs) {
ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor());
mDecoder->SetMediaDuration(durationUs);
}
}
if (mOmxDecoder->HasVideo()) {
int32_t displayWidth, displayHeight, width, height;
mOmxDecoder->GetVideoParameters(&displayWidth, &displayHeight,
&width, &height);
nsIntRect pictureRect(0, 0, width, height);
// Validate the container-reported frame and pictureRect sizes. This ensures
// that our video frame creation code doesn't overflow.
nsIntSize displaySize(displayWidth, displayHeight);
nsIntSize frameSize(width, height);
if (!IsValidVideoRegion(frameSize, pictureRect, displaySize)) {
return NS_ERROR_FAILURE;
}
// Video track's frame sizes will not overflow. Activate the video track.
mHasVideo = mInfo.mVideo.mHasVideo = true;
mInfo.mVideo.mDisplay = displaySize;
mPicture = pictureRect;
mInitialFrame = frameSize;
VideoFrameContainer* container = mDecoder->GetVideoFrameContainer();
if (container) {
container->SetCurrentFrame(gfxIntSize(displaySize.width, displaySize.height),
nullptr,
mozilla::TimeStamp::Now());
}
}
if (mOmxDecoder->HasAudio()) {
int32_t numChannels, sampleRate;
mOmxDecoder->GetAudioParameters(&numChannels, &sampleRate);
mHasAudio = mInfo.mAudio.mHasAudio = true;
mInfo.mAudio.mChannels = numChannels;
mInfo.mAudio.mRate = sampleRate;
}
*aInfo = mInfo;
#ifdef MOZ_AUDIO_OFFLOAD
CheckAudioOffload();
#endif
return NS_OK;
}
double HTMLVideoElement::MozFrameDelay()
{
MOZ_ASSERT(NS_IsMainThread(), "Should be on main thread.");
VideoFrameContainer* container = GetVideoFrameContainer();
return container ? container->GetFrameDelay() : 0;
}
NS_IMETHODIMP nsHTMLVideoElement::GetMozFrameDelay(double *aMozFrameDelay) {
NS_ASSERTION(NS_IsMainThread(), "Should be on main thread.");
VideoFrameContainer* container = GetVideoFrameContainer();
*aMozFrameDelay = container ? container->GetFrameDelay() : 0;
return NS_OK;
}
