MP4TrackDemuxer::MP4TrackDemuxer(MP4Demuxer* aParent, UniquePtr<TrackInfo>&& aInfo, const nsTArray<mp4_demuxer::Index::Indice>& indices) : mParent(aParent) , mStream(new mp4_demuxer::ResourceStream(mParent->mResource)) , mInfo(Move(aInfo)) , mIndex(new mp4_demuxer::Index(indices, mStream, mInfo->mTrackId, mInfo->IsAudio())) , mIterator(MakeUnique<mp4_demuxer::SampleIterator>(mIndex)) , mNeedReIndex(true) { EnsureUpToDateIndex(); // Force update of index // Collect telemetry from h264 AVCC SPS. if (mInfo->GetAsVideoInfo() && (mInfo->mMimeType.EqualsLiteral("video/mp4") || mInfo->mMimeType.EqualsLiteral("video/avc"))) { mNeedSPSForTelemetry = AccumulateSPSTelemetry(mInfo->GetAsVideoInfo()->mExtraData); } else { // No SPS to be found. mNeedSPSForTelemetry = false; } }
void MP4TrackDemuxer::UpdateSamples(nsTArray<RefPtr<MediaRawData>>& aSamples) { for (size_t i = 0; i < aSamples.Length(); i++) { MediaRawData* sample = aSamples[i]; // Collect telemetry from h264 Annex B SPS. if (mNeedSPSForTelemetry && mp4_demuxer::AnnexB::HasSPS(sample)) { RefPtr<MediaByteBuffer> extradata = mp4_demuxer::AnnexB::ExtractExtraData(sample); mNeedSPSForTelemetry = AccumulateSPSTelemetry(extradata); } if (sample->mCrypto.mValid) { nsAutoPtr<MediaRawDataWriter> writer(sample->CreateWriter()); writer->mCrypto.mMode = mInfo->mCrypto.mMode; writer->mCrypto.mIVSize = mInfo->mCrypto.mIVSize; writer->mCrypto.mKeyId.AppendElements(mInfo->mCrypto.mKeyId); } if (mInfo->GetAsVideoInfo()) { sample->mExtraData = mInfo->GetAsVideoInfo()->mExtraData; } } if (mNextKeyframeTime.isNothing() || aSamples.LastElement()->mTime >= mNextKeyframeTime.value().ToMicroseconds()) { SetNextKeyFrameTime(); } }
RefPtr<MP4TrackDemuxer::SamplesPromise> MP4TrackDemuxer::GetSamples(int32_t aNumSamples) { EnsureUpToDateIndex(); RefPtr<SamplesHolder> samples = new SamplesHolder; if (!aNumSamples) { return SamplesPromise::CreateAndReject(NS_ERROR_DOM_MEDIA_DEMUXER_ERR, __func__); } if (mQueuedSample) { MOZ_ASSERT(mQueuedSample->mKeyframe, "mQueuedSample must be a keyframe"); samples->mSamples.AppendElement(mQueuedSample); mQueuedSample = nullptr; aNumSamples--; } RefPtr<MediaRawData> sample; while (aNumSamples && (sample = GetNextSample())) { if (!sample->Size()) { continue; } samples->mSamples.AppendElement(sample); aNumSamples--; } if (samples->mSamples.IsEmpty()) { return SamplesPromise::CreateAndReject(NS_ERROR_DOM_MEDIA_END_OF_STREAM, __func__); } for (const auto& sample : samples->mSamples) { // Collect telemetry from h264 Annex B SPS. if (mNeedSPSForTelemetry && mp4_demuxer::AnnexB::HasSPS(sample)) { RefPtr<MediaByteBuffer> extradata = mp4_demuxer::AnnexB::ExtractExtraData(sample); mNeedSPSForTelemetry = AccumulateSPSTelemetry(extradata); } } if (mNextKeyframeTime.isNothing() || samples->mSamples.LastElement()->mTime >= mNextKeyframeTime.value()) { SetNextKeyFrameTime(); } return SamplesPromise::CreateAndResolve(samples, __func__); }
MP4TrackDemuxer::MP4TrackDemuxer(MP4Demuxer* aParent, UniquePtr<TrackInfo>&& aInfo, const mp4_demuxer::IndiceWrapper& aIndices) : mParent(aParent) , mStream(new mp4_demuxer::ResourceStream(mParent->mResource)) , mInfo(Move(aInfo)) , mIndex(new mp4_demuxer::Index(aIndices, mStream, mInfo->mTrackId, mInfo->IsAudio())) , mIterator(MakeUnique<mp4_demuxer::SampleIterator>(mIndex)) , mNeedReIndex(true) { EnsureUpToDateIndex(); // Force update of index VideoInfo* videoInfo = mInfo->GetAsVideoInfo(); // Collect telemetry from h264 AVCC SPS. if (videoInfo && (mInfo->mMimeType.EqualsLiteral("video/mp4") || mInfo->mMimeType.EqualsLiteral("video/avc"))) { mIsH264 = true; RefPtr<MediaByteBuffer> extraData = videoInfo->mExtraData; mNeedSPSForTelemetry = AccumulateSPSTelemetry(extraData); mp4_demuxer::SPSData spsdata; if (mp4_demuxer::H264::DecodeSPSFromExtraData(extraData, spsdata) && spsdata.pic_width > 0 && spsdata.pic_height > 0 && mp4_demuxer::H264::EnsureSPSIsSane(spsdata)) { videoInfo->mImage.width = spsdata.pic_width; videoInfo->mImage.height = spsdata.pic_height; videoInfo->mDisplay.width = spsdata.display_width; videoInfo->mDisplay.height = spsdata.display_height; } } else { // No SPS to be found. mNeedSPSForTelemetry = false; } }
void MP4Reader::Update(TrackType aTrack) { MOZ_ASSERT(GetTaskQueue()->IsCurrentThreadIn()); if (mShutdown) { return; } // Record number of frames decoded and parsed. Automatically update the // stats counters using the AutoNotifyDecoded stack-based class. AbstractMediaDecoder::AutoNotifyDecoded a(mDecoder); bool needInput = false; bool needOutput = false; auto& decoder = GetDecoderData(aTrack); { MonitorAutoLock lock(decoder.mMonitor); decoder.mUpdateScheduled = false; if (NeedInput(decoder)) { needInput = true; decoder.mInputExhausted = false; decoder.mNumSamplesInput++; } if (aTrack == kVideo) { uint64_t delta = decoder.mNumSamplesOutput - mLastReportedNumDecodedFrames; a.mDecoded = static_cast<uint32_t>(delta); mLastReportedNumDecodedFrames = decoder.mNumSamplesOutput; } if (decoder.HasPromise()) { needOutput = true; if (!decoder.mOutput.IsEmpty()) { nsRefPtr<MediaData> output = decoder.mOutput[0]; decoder.mOutput.RemoveElementAt(0); ReturnOutput(output, aTrack); } else if (decoder.mDrainComplete) { decoder.RejectPromise(END_OF_STREAM, __func__); } } } VLOG("Update(%s) ni=%d no=%d iex=%d fl=%d", TrackTypeToStr(aTrack), needInput, needOutput, decoder.mInputExhausted, decoder.mIsFlushing); if (needInput) { nsAutoPtr<MediaSample> sample(PopSample(aTrack)); // Collect telemetry from h264 Annex B SPS. if (!mFoundSPSForTelemetry && sample && AnnexB::HasSPS(sample->mMp4Sample)) { nsRefPtr<ByteBuffer> extradata = AnnexB::ExtractExtraData(sample->mMp4Sample); mFoundSPSForTelemetry = AccumulateSPSTelemetry(extradata); } if (sample) { decoder.mDecoder->Input(sample->mMp4Sample.forget()); if (aTrack == kVideo) { a.mParsed++; } } else { { MonitorAutoLock lock(decoder.mMonitor); MOZ_ASSERT(!decoder.mDemuxEOS); decoder.mDemuxEOS = true; } // DrainComplete takes care of reporting EOS upwards decoder.mDecoder->Drain(); } } }
nsresult MP4Reader::ReadMetadata(MediaInfo* aInfo, MetadataTags** aTags) { if (!mDemuxerInitialized) { MonitorAutoLock mon(mDemuxerMonitor); bool ok = InvokeAndRetry(this, &MP4Reader::InitDemuxer, mStream, &mDemuxerMonitor); NS_ENSURE_TRUE(ok, NS_ERROR_FAILURE); mIndexReady = true; // To decode, we need valid video and a place to put it. mInfo.mVideo.mHasVideo = mVideo.mActive = mDemuxer->HasValidVideo() && mDecoder->GetImageContainer(); if (mVideo.mActive) { mVideo.mTrackDemuxer = new MP4VideoDemuxer(mDemuxer); } mInfo.mAudio.mHasAudio = mAudio.mActive = mDemuxer->HasValidAudio(); if (mAudio.mActive) { mAudio.mTrackDemuxer = new MP4AudioDemuxer(mDemuxer); } mCrypto = mDemuxer->Crypto(); { MonitorAutoUnlock unlock(mDemuxerMonitor); ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor()); mInfo.mCrypto.mIsEncrypted = mIsEncrypted = mCrypto.valid; } // Remember that we've initialized the demuxer, so that if we're decoding // an encrypted stream and we need to wait for a CDM to be set, we don't // need to reinit the demuxer. mDemuxerInitialized = true; } else if (mPlatform && !IsWaitingMediaResources()) { *aInfo = mInfo; *aTags = nullptr; return NS_OK; } if (HasAudio()) { const AudioDecoderConfig& audio = mDemuxer->AudioConfig(); mInfo.mAudio.mRate = audio.samples_per_second; mInfo.mAudio.mChannels = audio.channel_count; mAudio.mCallback = new DecoderCallback(this, kAudio); } if (HasVideo()) { const VideoDecoderConfig& video = mDemuxer->VideoConfig(); mInfo.mVideo.mDisplay = nsIntSize(video.display_width, video.display_height); mVideo.mCallback = new DecoderCallback(this, kVideo); // Collect telemetry from h264 AVCC SPS. if (!mFoundSPSForTelemetry) { mFoundSPSForTelemetry = AccumulateSPSTelemetry(video.extra_data); } } if (mIsEncrypted) { nsTArray<uint8_t> initData; ExtractCryptoInitData(initData); if (initData.Length() == 0) { return NS_ERROR_FAILURE; } mInfo.mCrypto.mInitData = initData; mInfo.mCrypto.mType = NS_LITERAL_STRING("cenc"); } // Get the duration, and report it to the decoder if we have it. Microseconds duration; { MonitorAutoLock lock(mDemuxerMonitor); duration = mDemuxer->Duration(); } if (duration != -1) { ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor()); mDecoder->SetMediaDuration(duration); } *aInfo = mInfo; *aTags = nullptr; if (!IsWaitingMediaResources() && !IsWaitingOnCDMResource()) { NS_ENSURE_TRUE(EnsureDecodersSetup(), NS_ERROR_FAILURE); } MonitorAutoLock mon(mDemuxerMonitor); UpdateIndex(); return NS_OK; }
nsresult MP4Reader::ReadMetadata(MediaInfo* aInfo, MetadataTags** aTags) { if (!mDemuxerInitialized) { MonitorAutoLock mon(mDemuxerMonitor); bool ok = InvokeAndRetry(this, &MP4Reader::InitDemuxer, mStream, &mDemuxerMonitor); NS_ENSURE_TRUE(ok, NS_ERROR_FAILURE); mIndexReady = true; // To decode, we need valid video and a place to put it. mVideo.mActive = mDemuxer->HasValidVideo() && mDecoder->GetImageContainer(); if (mVideo.mActive) { mVideo.mTrackDemuxer = new MP4VideoDemuxer(mDemuxer); } mAudio.mActive = mDemuxer->HasValidAudio(); if (mAudio.mActive) { mAudio.mTrackDemuxer = new MP4AudioDemuxer(mDemuxer); } mCrypto = mDemuxer->Crypto(); { MonitorAutoUnlock unlock(mDemuxerMonitor); ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor()); mIsEncrypted = mCrypto.valid; } // Remember that we've initialized the demuxer, so that if we're decoding // an encrypted stream and we need to wait for a CDM to be set, we don't // need to reinit the demuxer. mDemuxerInitialized = true; } else if (mPlatform && !IsWaitingMediaResources()) { *aInfo = mInfo; *aTags = nullptr; } if (HasAudio()) { mInfo.mAudio = mDemuxer->AudioConfig(); mAudio.mCallback = new DecoderCallback(this, TrackInfo::kAudioTrack); } if (HasVideo()) { mInfo.mVideo = mDemuxer->VideoConfig(); mVideo.mCallback = new DecoderCallback(this, TrackInfo::kVideoTrack); // Collect telemetry from h264 AVCC SPS. if (!mFoundSPSForTelemetry) { mFoundSPSForTelemetry = AccumulateSPSTelemetry(mInfo.mVideo.mExtraData); } } if (mCrypto.valid) { nsTArray<uint8_t> initData; ExtractCryptoInitData(initData); if (initData.Length() == 0) { return NS_ERROR_FAILURE; } // Add init data to info, will get sent from HTMLMediaElement::MetadataLoaded // (i.e., when transitioning from HAVE_NOTHING to HAVE_METADATA). mInfo.mCrypto.AddInitData(NS_LITERAL_STRING("cenc"), Move(initData)); } // Get the duration, and report it to the decoder if we have it. Microseconds duration; { MonitorAutoLock lock(mDemuxerMonitor); duration = mDemuxer->Duration(); } if (duration != -1) { ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor()); mDecoder->SetMediaDuration(duration); } *aInfo = mInfo; *aTags = nullptr; if (!IsWaitingOnCDMResource()) { NS_ENSURE_TRUE(EnsureDecodersSetup(), NS_ERROR_FAILURE); } MonitorAutoLock mon(mDemuxerMonitor); UpdateIndex(); return NS_OK; }