void
MP4Reader::Output(TrackType aTrack, MediaData* aSample)
{
#ifdef LOG_SAMPLE_DECODE
VLOG("Decoded %s sample time=%lld dur=%lld",
TrackTypeToStr(aTrack), aSample->mTime, aSample->mDuration);
#endif
if (!aSample) {
NS_WARNING("MP4Reader::Output() passed a null sample");
Error(aTrack);
return;
}
auto& decoder = GetDecoderData(aTrack);
// Don't accept output while we're flushing.
MonitorAutoLock mon(decoder.mMonitor);
if (decoder.mIsFlushing) {
LOG("MP4Reader produced output while flushing, discarding.");
mon.NotifyAll();
return;
}
decoder.mOutput.AppendElement(aSample);
decoder.mNumSamplesOutput++;
if (NeedInput(decoder) || decoder.HasPromise()) {
ScheduleUpdate(aTrack);
}
}
void
MP4Reader::ScheduleUpdate(TrackType aTrack)
{
auto& decoder = GetDecoderData(aTrack);
decoder.mMonitor.AssertCurrentThreadOwns();
if (decoder.mUpdateScheduled) {
return;
}
VLOG("SchedulingUpdate(%s)", TrackTypeToStr(aTrack));
decoder.mUpdateScheduled = true;
RefPtr<nsIRunnable> task(
NS_NewRunnableMethodWithArg<TrackType>(this, &MP4Reader::Update, aTrack));
GetTaskQueue()->Dispatch(task.forget());
}
void
MP4Reader::Flush(TrackType aTrack)
{
MOZ_ASSERT(GetTaskQueue()->IsCurrentThreadIn());
VLOG("Flush(%s) BEGIN", TrackTypeToStr(aTrack));
DecoderData& data = GetDecoderData(aTrack);
if (!data.mDecoder) {
return;
}
// Purge the current decoder's state.
// Set a flag so that we ignore all output while we call
// MediaDataDecoder::Flush().
{
MonitorAutoLock mon(data.mMonitor);
data.mIsFlushing = true;
data.mDemuxEOS = false;
data.mDrainComplete = false;
}
data.mDecoder->Flush();
{
MonitorAutoLock mon(data.mMonitor);
data.mIsFlushing = false;
data.mOutput.Clear();
data.mNumSamplesInput = 0;
data.mNumSamplesOutput = 0;
data.mInputExhausted = false;
if (data.HasPromise()) {
data.RejectPromise(CANCELED, __func__);
}
data.mDiscontinuity = true;
data.mUpdateScheduled = false;
}
if (aTrack == kVideo) {
mQueuedVideoSample = nullptr;
}
VLOG("Flush(%s) END", TrackTypeToStr(aTrack));
}
/* static */ RefPtr<AllocationWrapper::AllocateDecoderPromise>
AllocationWrapper::CreateDecoder(const CreateDecoderParams& aParams)
{
// aParams.mConfig is guaranteed to stay alive during the lifetime of the
// MediaDataDecoder, so keeping a pointer to the object is safe.
const TrackInfo* config = &aParams.mConfig;
RefPtr<TaskQueue> taskQueue = aParams.mTaskQueue;
DecoderDoctorDiagnostics* diagnostics = aParams.mDiagnostics;
RefPtr<layers::ImageContainer> imageContainer = aParams.mImageContainer;
RefPtr<layers::KnowsCompositor> knowsCompositor = aParams.mKnowsCompositor;
RefPtr<GMPCrashHelper> crashHelper = aParams.mCrashHelper;
CreateDecoderParams::UseNullDecoder useNullDecoder = aParams.mUseNullDecoder;
CreateDecoderParams::NoWrapper noWrapper = aParams.mNoWrapper;
TrackInfo::TrackType type = aParams.mType;
MediaEventProducer<TrackInfo::TrackType>* onWaitingForKeyEvent =
aParams.mOnWaitingForKeyEvent;
CreateDecoderParams::OptionSet options = aParams.mOptions;
CreateDecoderParams::VideoFrameRate rate = aParams.mRate;
RefPtr<AllocateDecoderPromise> p =
GlobalAllocPolicy::Instance(aParams.mType)
.Alloc()
->Then(
AbstractThread::GetCurrent(),
__func__,
[=](RefPtr<Token> aToken) {
// result may not always be updated by PDMFactory::CreateDecoder
// either when the creation succeeded or failed, as such it must be
// initialized to a fatal error by default.
MediaResult result = MediaResult(
NS_ERROR_DOM_MEDIA_FATAL_ERR,
nsPrintfCString("error creating %s decoder", TrackTypeToStr(type)));
RefPtr<PDMFactory> pdm = new PDMFactory();
CreateDecoderParams params{ *config,
taskQueue,
diagnostics,
imageContainer,
&result,
knowsCompositor,
crashHelper,
useNullDecoder,
noWrapper,
type,
onWaitingForKeyEvent,
options,
rate };
RefPtr<MediaDataDecoder> decoder = pdm->CreateDecoder(params);
if (decoder) {
RefPtr<AllocationWrapper> wrapper =
new AllocationWrapper(decoder.forget(), aToken.forget());
return AllocateDecoderPromise::CreateAndResolve(wrapper, __func__);
}
return AllocateDecoderPromise::CreateAndReject(result, __func__);
},
[]() {
return AllocateDecoderPromise::CreateAndReject(
MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR,
"Allocation policy expired"),
__func__);
});
return p;
}
MP4Metadata::ResultAndTrackInfo MP4Metadata::GetTrackInfo(
mozilla::TrackInfo::TrackType aType, size_t aTrackNumber) const {
Maybe<uint32_t> trackIndex = TrackTypeToGlobalTrackIndex(aType, aTrackNumber);
if (trackIndex.isNothing()) {
return {MediaResult(NS_ERROR_DOM_MEDIA_METADATA_ERR,
RESULT_DETAIL("No %s tracks", TrackTypeToStr(aType))),
nullptr};
}
Mp4parseTrackInfo info;
auto rv = mp4parse_get_track_info(mParser.get(), trackIndex.value(), &info);
if (rv != MP4PARSE_STATUS_OK) {
MOZ_LOG(gMP4MetadataLog, LogLevel::Warning,
("mp4parse_get_track_info returned %d", rv));
return {MediaResult(NS_ERROR_DOM_MEDIA_METADATA_ERR,
RESULT_DETAIL("Cannot find %s track #%zu",
TrackTypeToStr(aType), aTrackNumber)),
nullptr};
}
#ifdef DEBUG
bool haveSampleInfo = false;
const char* codecString = "unrecognized";
Mp4parseCodec codecType = MP4PARSE_CODEC_UNKNOWN;
if (info.track_type == MP4PARSE_TRACK_TYPE_AUDIO) {
Mp4parseTrackAudioInfo audio;
auto rv = mp4parse_get_track_audio_info(mParser.get(), trackIndex.value(),
&audio);
if (rv == MP4PARSE_STATUS_OK && audio.sample_info_count > 0) {
codecType = audio.sample_info[0].codec_type;
haveSampleInfo = true;
}
} else if (info.track_type == MP4PARSE_TRACK_TYPE_VIDEO) {
Mp4parseTrackVideoInfo video;
auto rv = mp4parse_get_track_video_info(mParser.get(), trackIndex.value(),
&video);
if (rv == MP4PARSE_STATUS_OK && video.sample_info_count > 0) {
codecType = video.sample_info[0].codec_type;
haveSampleInfo = true;
}
}
if (haveSampleInfo) {
switch (codecType) {
case MP4PARSE_CODEC_UNKNOWN:
codecString = "unknown";
break;
case MP4PARSE_CODEC_AAC:
codecString = "aac";
break;
case MP4PARSE_CODEC_OPUS:
codecString = "opus";
break;
case MP4PARSE_CODEC_FLAC:
codecString = "flac";
break;
case MP4PARSE_CODEC_ALAC:
codecString = "alac";
break;
case MP4PARSE_CODEC_AVC:
codecString = "h.264";
break;
case MP4PARSE_CODEC_VP9:
codecString = "vp9";
break;
case MP4PARSE_CODEC_AV1:
codecString = "av1";
break;
case MP4PARSE_CODEC_MP3:
codecString = "mp3";
break;
case MP4PARSE_CODEC_MP4V:
codecString = "mp4v";
break;
case MP4PARSE_CODEC_JPEG:
codecString = "jpeg";
break;
case MP4PARSE_CODEC_AC3:
codecString = "ac-3";
break;
case MP4PARSE_CODEC_EC3:
codecString = "ec-3";
break;
}
}
MOZ_LOG(gMP4MetadataLog, LogLevel::Debug,
("track codec %s (%u)\n", codecString, codecType));
#endif
// This specialization interface is crazy.
UniquePtr<mozilla::TrackInfo> e;
switch (aType) {
case TrackInfo::TrackType::kAudioTrack: {
Mp4parseTrackAudioInfo audio;
auto rv = mp4parse_get_track_audio_info(mParser.get(), trackIndex.value(),
&audio);
if (rv != MP4PARSE_STATUS_OK) {
MOZ_LOG(gMP4MetadataLog, LogLevel::Warning,
("mp4parse_get_track_audio_info returned error %d", rv));
return {MediaResult(NS_ERROR_DOM_MEDIA_METADATA_ERR,
RESULT_DETAIL("Cannot parse %s track #%zu",
TrackTypeToStr(aType), aTrackNumber)),
nullptr};
}
auto track = mozilla::MakeUnique<MP4AudioInfo>();
MediaResult updateStatus = track->Update(&info, &audio);
if (NS_FAILED(updateStatus)) {
MOZ_LOG(gMP4MetadataLog, LogLevel::Warning,
("Updating audio track failed with %s",
updateStatus.Message().get()));
return {MediaResult(NS_ERROR_DOM_MEDIA_METADATA_ERR,
RESULT_DETAIL(
"Failed to update %s track #%zu with error: %s",
TrackTypeToStr(aType), aTrackNumber,
updateStatus.Message().get())),
nullptr};
}
e = std::move(track);
} break;
case TrackInfo::TrackType::kVideoTrack: {
Mp4parseTrackVideoInfo video;
auto rv = mp4parse_get_track_video_info(mParser.get(), trackIndex.value(),
&video);
if (rv != MP4PARSE_STATUS_OK) {
MOZ_LOG(gMP4MetadataLog, LogLevel::Warning,
("mp4parse_get_track_video_info returned error %d", rv));
return {MediaResult(NS_ERROR_DOM_MEDIA_METADATA_ERR,
RESULT_DETAIL("Cannot parse %s track #%zu",
TrackTypeToStr(aType), aTrackNumber)),
nullptr};
}
auto track = mozilla::MakeUnique<MP4VideoInfo>();
MediaResult updateStatus = track->Update(&info, &video);
if (NS_FAILED(updateStatus)) {
MOZ_LOG(gMP4MetadataLog, LogLevel::Warning,
("Updating video track failed with %s",
updateStatus.Message().get()));
return {MediaResult(NS_ERROR_DOM_MEDIA_METADATA_ERR,
RESULT_DETAIL(
"Failed to update %s track #%zu with error: %s",
TrackTypeToStr(aType), aTrackNumber,
updateStatus.Message().get())),
nullptr};
}
e = std::move(track);
} break;
default:
MOZ_LOG(gMP4MetadataLog, LogLevel::Warning,
("unhandled track type %d", aType));
return {MediaResult(NS_ERROR_DOM_MEDIA_METADATA_ERR,
RESULT_DETAIL("Cannot handle %s track #%zu",
TrackTypeToStr(aType), aTrackNumber)),
nullptr};
}
// No duration in track, use fragment_duration.
if (e && !e->mDuration.IsPositive()) {
Mp4parseFragmentInfo info;
auto rv = mp4parse_get_fragment_info(mParser.get(), &info);
if (rv == MP4PARSE_STATUS_OK) {
e->mDuration = TimeUnit::FromMicroseconds(info.fragment_duration);
}
}
if (e && e->IsValid()) {
return {NS_OK, std::move(e)};
}
MOZ_LOG(gMP4MetadataLog, LogLevel::Debug, ("TrackInfo didn't validate"));
return {MediaResult(NS_ERROR_DOM_MEDIA_METADATA_ERR,
RESULT_DETAIL("Invalid %s track #%zu",
TrackTypeToStr(aType), aTrackNumber)),
nullptr};
}
void
MP4Reader::Update(TrackType aTrack)
{
MOZ_ASSERT(GetTaskQueue()->IsCurrentThreadIn());
if (mShutdown) {
return;
}
// Record number of frames decoded and parsed. Automatically update the
// stats counters using the AutoNotifyDecoded stack-based class.
AbstractMediaDecoder::AutoNotifyDecoded a(mDecoder);
bool needInput = false;
bool needOutput = false;
auto& decoder = GetDecoderData(aTrack);
{
MonitorAutoLock lock(decoder.mMonitor);
decoder.mUpdateScheduled = false;
if (NeedInput(decoder)) {
needInput = true;
decoder.mInputExhausted = false;
decoder.mNumSamplesInput++;
}
if (aTrack == kVideo) {
uint64_t delta = decoder.mNumSamplesOutput - mLastReportedNumDecodedFrames;
a.mDecoded = static_cast<uint32_t>(delta);
mLastReportedNumDecodedFrames = decoder.mNumSamplesOutput;
}
if (decoder.HasPromise()) {
needOutput = true;
if (!decoder.mOutput.IsEmpty()) {
nsRefPtr<MediaData> output = decoder.mOutput[0];
decoder.mOutput.RemoveElementAt(0);
ReturnOutput(output, aTrack);
} else if (decoder.mDrainComplete) {
decoder.RejectPromise(END_OF_STREAM, __func__);
}
}
}
VLOG("Update(%s) ni=%d no=%d iex=%d fl=%d",
TrackTypeToStr(aTrack),
needInput,
needOutput,
decoder.mInputExhausted,
decoder.mIsFlushing);
if (needInput) {
nsAutoPtr<MediaSample> sample(PopSample(aTrack));
// Collect telemetry from h264 Annex B SPS.
if (!mFoundSPSForTelemetry && sample && AnnexB::HasSPS(sample->mMp4Sample)) {
nsRefPtr<ByteBuffer> extradata = AnnexB::ExtractExtraData(sample->mMp4Sample);
mFoundSPSForTelemetry = AccumulateSPSTelemetry(extradata);
}
if (sample) {
decoder.mDecoder->Input(sample->mMp4Sample.forget());
if (aTrack == kVideo) {
a.mParsed++;
}
} else {
{
MonitorAutoLock lock(decoder.mMonitor);
MOZ_ASSERT(!decoder.mDemuxEOS);
decoder.mDemuxEOS = true;
}
// DrainComplete takes care of reporting EOS upwards
decoder.mDecoder->Drain();
}
}
}