MP4TrackDemuxer::MP4TrackDemuxer(MP4Demuxer* aParent,
UniquePtr<TrackInfo>&& aInfo,
const nsTArray<mp4_demuxer::Index::Indice>& indices)
: mParent(aParent)
, mStream(new mp4_demuxer::ResourceStream(mParent->mResource))
, mInfo(Move(aInfo))
, mIndex(new mp4_demuxer::Index(indices,
mStream,
mInfo->mTrackId,
mInfo->IsAudio()))
, mIterator(MakeUnique<mp4_demuxer::SampleIterator>(mIndex))
, mNeedReIndex(true)
{
EnsureUpToDateIndex(); // Force update of index
// Collect telemetry from h264 AVCC SPS.
if (mInfo->GetAsVideoInfo() &&
(mInfo->mMimeType.EqualsLiteral("video/mp4") ||
mInfo->mMimeType.EqualsLiteral("video/avc"))) {
mNeedSPSForTelemetry =
AccumulateSPSTelemetry(mInfo->GetAsVideoInfo()->mExtraData);
} else {
// No SPS to be found.
mNeedSPSForTelemetry = false;
}
}
void
MP4TrackDemuxer::UpdateSamples(nsTArray<RefPtr<MediaRawData>>& aSamples)
{
for (size_t i = 0; i < aSamples.Length(); i++) {
MediaRawData* sample = aSamples[i];
// Collect telemetry from h264 Annex B SPS.
if (mNeedSPSForTelemetry && mp4_demuxer::AnnexB::HasSPS(sample)) {
RefPtr<MediaByteBuffer> extradata =
mp4_demuxer::AnnexB::ExtractExtraData(sample);
mNeedSPSForTelemetry = AccumulateSPSTelemetry(extradata);
}
if (sample->mCrypto.mValid) {
nsAutoPtr<MediaRawDataWriter> writer(sample->CreateWriter());
writer->mCrypto.mMode = mInfo->mCrypto.mMode;
writer->mCrypto.mIVSize = mInfo->mCrypto.mIVSize;
writer->mCrypto.mKeyId.AppendElements(mInfo->mCrypto.mKeyId);
}
if (mInfo->GetAsVideoInfo()) {
sample->mExtraData = mInfo->GetAsVideoInfo()->mExtraData;
}
}
if (mNextKeyframeTime.isNothing() ||
aSamples.LastElement()->mTime >= mNextKeyframeTime.value().ToMicroseconds()) {
SetNextKeyFrameTime();
}
}
RefPtr<MP4TrackDemuxer::SamplesPromise>
MP4TrackDemuxer::GetSamples(int32_t aNumSamples)
{
EnsureUpToDateIndex();
RefPtr<SamplesHolder> samples = new SamplesHolder;
if (!aNumSamples) {
return SamplesPromise::CreateAndReject(NS_ERROR_DOM_MEDIA_DEMUXER_ERR,
__func__);
}
if (mQueuedSample) {
MOZ_ASSERT(mQueuedSample->mKeyframe,
"mQueuedSample must be a keyframe");
samples->mSamples.AppendElement(mQueuedSample);
mQueuedSample = nullptr;
aNumSamples--;
}
RefPtr<MediaRawData> sample;
while (aNumSamples && (sample = GetNextSample())) {
if (!sample->Size()) {
continue;
}
samples->mSamples.AppendElement(sample);
aNumSamples--;
}
if (samples->mSamples.IsEmpty()) {
return SamplesPromise::CreateAndReject(NS_ERROR_DOM_MEDIA_END_OF_STREAM,
__func__);
}
for (const auto& sample : samples->mSamples) {
// Collect telemetry from h264 Annex B SPS.
if (mNeedSPSForTelemetry && mp4_demuxer::AnnexB::HasSPS(sample)) {
RefPtr<MediaByteBuffer> extradata =
mp4_demuxer::AnnexB::ExtractExtraData(sample);
mNeedSPSForTelemetry = AccumulateSPSTelemetry(extradata);
}
}
if (mNextKeyframeTime.isNothing()
|| samples->mSamples.LastElement()->mTime
>= mNextKeyframeTime.value()) {
SetNextKeyFrameTime();
}
return SamplesPromise::CreateAndResolve(samples, __func__);
}
MP4TrackDemuxer::MP4TrackDemuxer(MP4Demuxer* aParent,
UniquePtr<TrackInfo>&& aInfo,
const mp4_demuxer::IndiceWrapper& aIndices)
: mParent(aParent)
, mStream(new mp4_demuxer::ResourceStream(mParent->mResource))
, mInfo(Move(aInfo))
, mIndex(new mp4_demuxer::Index(aIndices,
mStream,
mInfo->mTrackId,
mInfo->IsAudio()))
, mIterator(MakeUnique<mp4_demuxer::SampleIterator>(mIndex))
, mNeedReIndex(true)
{
EnsureUpToDateIndex(); // Force update of index
VideoInfo* videoInfo = mInfo->GetAsVideoInfo();
// Collect telemetry from h264 AVCC SPS.
if (videoInfo
&& (mInfo->mMimeType.EqualsLiteral("video/mp4")
|| mInfo->mMimeType.EqualsLiteral("video/avc"))) {
mIsH264 = true;
RefPtr<MediaByteBuffer> extraData = videoInfo->mExtraData;
mNeedSPSForTelemetry = AccumulateSPSTelemetry(extraData);
mp4_demuxer::SPSData spsdata;
if (mp4_demuxer::H264::DecodeSPSFromExtraData(extraData, spsdata)
&& spsdata.pic_width > 0 && spsdata.pic_height > 0
&& mp4_demuxer::H264::EnsureSPSIsSane(spsdata)) {
videoInfo->mImage.width = spsdata.pic_width;
videoInfo->mImage.height = spsdata.pic_height;
videoInfo->mDisplay.width = spsdata.display_width;
videoInfo->mDisplay.height = spsdata.display_height;
}
} else {
// No SPS to be found.
mNeedSPSForTelemetry = false;
}
}
void
MP4Reader::Update(TrackType aTrack)
{
MOZ_ASSERT(GetTaskQueue()->IsCurrentThreadIn());
if (mShutdown) {
return;
}
// Record number of frames decoded and parsed. Automatically update the
// stats counters using the AutoNotifyDecoded stack-based class.
AbstractMediaDecoder::AutoNotifyDecoded a(mDecoder);
bool needInput = false;
bool needOutput = false;
auto& decoder = GetDecoderData(aTrack);
{
MonitorAutoLock lock(decoder.mMonitor);
decoder.mUpdateScheduled = false;
if (NeedInput(decoder)) {
needInput = true;
decoder.mInputExhausted = false;
decoder.mNumSamplesInput++;
}
if (aTrack == kVideo) {
uint64_t delta = decoder.mNumSamplesOutput - mLastReportedNumDecodedFrames;
a.mDecoded = static_cast<uint32_t>(delta);
mLastReportedNumDecodedFrames = decoder.mNumSamplesOutput;
}
if (decoder.HasPromise()) {
needOutput = true;
if (!decoder.mOutput.IsEmpty()) {
nsRefPtr<MediaData> output = decoder.mOutput[0];
decoder.mOutput.RemoveElementAt(0);
ReturnOutput(output, aTrack);
} else if (decoder.mDrainComplete) {
decoder.RejectPromise(END_OF_STREAM, __func__);
}
}
}
VLOG("Update(%s) ni=%d no=%d iex=%d fl=%d",
TrackTypeToStr(aTrack),
needInput,
needOutput,
decoder.mInputExhausted,
decoder.mIsFlushing);
if (needInput) {
nsAutoPtr<MediaSample> sample(PopSample(aTrack));
// Collect telemetry from h264 Annex B SPS.
if (!mFoundSPSForTelemetry && sample && AnnexB::HasSPS(sample->mMp4Sample)) {
nsRefPtr<ByteBuffer> extradata = AnnexB::ExtractExtraData(sample->mMp4Sample);
mFoundSPSForTelemetry = AccumulateSPSTelemetry(extradata);
}
if (sample) {
decoder.mDecoder->Input(sample->mMp4Sample.forget());
if (aTrack == kVideo) {
a.mParsed++;
}
} else {
{
MonitorAutoLock lock(decoder.mMonitor);
MOZ_ASSERT(!decoder.mDemuxEOS);
decoder.mDemuxEOS = true;
}
// DrainComplete takes care of reporting EOS upwards
decoder.mDecoder->Drain();
}
}
}
nsresult
MP4Reader::ReadMetadata(MediaInfo* aInfo,
MetadataTags** aTags)
{
if (!mDemuxerInitialized) {
MonitorAutoLock mon(mDemuxerMonitor);
bool ok = InvokeAndRetry(this, &MP4Reader::InitDemuxer, mStream, &mDemuxerMonitor);
NS_ENSURE_TRUE(ok, NS_ERROR_FAILURE);
mIndexReady = true;
// To decode, we need valid video and a place to put it.
mInfo.mVideo.mHasVideo = mVideo.mActive = mDemuxer->HasValidVideo() &&
mDecoder->GetImageContainer();
if (mVideo.mActive) {
mVideo.mTrackDemuxer = new MP4VideoDemuxer(mDemuxer);
}
mInfo.mAudio.mHasAudio = mAudio.mActive = mDemuxer->HasValidAudio();
if (mAudio.mActive) {
mAudio.mTrackDemuxer = new MP4AudioDemuxer(mDemuxer);
}
mCrypto = mDemuxer->Crypto();
{
MonitorAutoUnlock unlock(mDemuxerMonitor);
ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor());
mInfo.mCrypto.mIsEncrypted = mIsEncrypted = mCrypto.valid;
}
// Remember that we've initialized the demuxer, so that if we're decoding
// an encrypted stream and we need to wait for a CDM to be set, we don't
// need to reinit the demuxer.
mDemuxerInitialized = true;
} else if (mPlatform && !IsWaitingMediaResources()) {
*aInfo = mInfo;
*aTags = nullptr;
return NS_OK;
}
if (HasAudio()) {
const AudioDecoderConfig& audio = mDemuxer->AudioConfig();
mInfo.mAudio.mRate = audio.samples_per_second;
mInfo.mAudio.mChannels = audio.channel_count;
mAudio.mCallback = new DecoderCallback(this, kAudio);
}
if (HasVideo()) {
const VideoDecoderConfig& video = mDemuxer->VideoConfig();
mInfo.mVideo.mDisplay =
nsIntSize(video.display_width, video.display_height);
mVideo.mCallback = new DecoderCallback(this, kVideo);
// Collect telemetry from h264 AVCC SPS.
if (!mFoundSPSForTelemetry) {
mFoundSPSForTelemetry = AccumulateSPSTelemetry(video.extra_data);
}
}
if (mIsEncrypted) {
nsTArray<uint8_t> initData;
ExtractCryptoInitData(initData);
if (initData.Length() == 0) {
return NS_ERROR_FAILURE;
}
mInfo.mCrypto.mInitData = initData;
mInfo.mCrypto.mType = NS_LITERAL_STRING("cenc");
}
// Get the duration, and report it to the decoder if we have it.
Microseconds duration;
{
MonitorAutoLock lock(mDemuxerMonitor);
duration = mDemuxer->Duration();
}
if (duration != -1) {
ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor());
mDecoder->SetMediaDuration(duration);
}
*aInfo = mInfo;
*aTags = nullptr;
if (!IsWaitingMediaResources() && !IsWaitingOnCDMResource()) {
NS_ENSURE_TRUE(EnsureDecodersSetup(), NS_ERROR_FAILURE);
}
MonitorAutoLock mon(mDemuxerMonitor);
UpdateIndex();
return NS_OK;
}
nsresult
MP4Reader::ReadMetadata(MediaInfo* aInfo,
MetadataTags** aTags)
{
if (!mDemuxerInitialized) {
MonitorAutoLock mon(mDemuxerMonitor);
bool ok = InvokeAndRetry(this, &MP4Reader::InitDemuxer, mStream, &mDemuxerMonitor);
NS_ENSURE_TRUE(ok, NS_ERROR_FAILURE);
mIndexReady = true;
// To decode, we need valid video and a place to put it.
mVideo.mActive = mDemuxer->HasValidVideo() && mDecoder->GetImageContainer();
if (mVideo.mActive) {
mVideo.mTrackDemuxer = new MP4VideoDemuxer(mDemuxer);
}
mAudio.mActive = mDemuxer->HasValidAudio();
if (mAudio.mActive) {
mAudio.mTrackDemuxer = new MP4AudioDemuxer(mDemuxer);
}
mCrypto = mDemuxer->Crypto();
{
MonitorAutoUnlock unlock(mDemuxerMonitor);
ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor());
mIsEncrypted = mCrypto.valid;
}
// Remember that we've initialized the demuxer, so that if we're decoding
// an encrypted stream and we need to wait for a CDM to be set, we don't
// need to reinit the demuxer.
mDemuxerInitialized = true;
} else if (mPlatform && !IsWaitingMediaResources()) {
*aInfo = mInfo;
*aTags = nullptr;
}
if (HasAudio()) {
mInfo.mAudio = mDemuxer->AudioConfig();
mAudio.mCallback = new DecoderCallback(this, TrackInfo::kAudioTrack);
}
if (HasVideo()) {
mInfo.mVideo = mDemuxer->VideoConfig();
mVideo.mCallback = new DecoderCallback(this, TrackInfo::kVideoTrack);
// Collect telemetry from h264 AVCC SPS.
if (!mFoundSPSForTelemetry) {
mFoundSPSForTelemetry = AccumulateSPSTelemetry(mInfo.mVideo.mExtraData);
}
}
if (mCrypto.valid) {
nsTArray<uint8_t> initData;
ExtractCryptoInitData(initData);
if (initData.Length() == 0) {
return NS_ERROR_FAILURE;
}
// Add init data to info, will get sent from HTMLMediaElement::MetadataLoaded
// (i.e., when transitioning from HAVE_NOTHING to HAVE_METADATA).
mInfo.mCrypto.AddInitData(NS_LITERAL_STRING("cenc"), Move(initData));
}
// Get the duration, and report it to the decoder if we have it.
Microseconds duration;
{
MonitorAutoLock lock(mDemuxerMonitor);
duration = mDemuxer->Duration();
}
if (duration != -1) {
ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor());
mDecoder->SetMediaDuration(duration);
}
*aInfo = mInfo;
*aTags = nullptr;
if (!IsWaitingOnCDMResource()) {
NS_ENSURE_TRUE(EnsureDecodersSetup(), NS_ERROR_FAILURE);
}
MonitorAutoLock mon(mDemuxerMonitor);
UpdateIndex();
return NS_OK;
}
