void
AudioNodeStream::ProduceOutputBeforeInput(GraphTime aFrom)
{
MOZ_ASSERT(mEngine->AsDelayNodeEngine());
MOZ_ASSERT(mEngine->OutputCount() == 1,
"DelayNodeEngine output count should be 1");
MOZ_ASSERT(!InMutedCycle(), "DelayNodes should break cycles");
MOZ_ASSERT(mLastChunks.Length() == 1);
// Consider this stream blocked if it has already finished output. Normally
// mBlocked would reflect this, but due to rounding errors our audio track may
// appear to extend slightly beyond aFrom, so we might not be blocked yet.
bool blocked = mFinished || mBlocked.GetAt(aFrom);
// If the stream has finished at this time, it will be blocked.
if (blocked) {
mLastChunks[0].SetNull(WEBAUDIO_BLOCK_SIZE);
} else {
mEngine->ProduceBlockBeforeInput(&mLastChunks[0]);
NS_ASSERTION(mLastChunks[0].GetDuration() == WEBAUDIO_BLOCK_SIZE,
"Invalid WebAudio chunk size");
if (mDisabledTrackIDs.Contains(static_cast<TrackID>(AUDIO_TRACK))) {
mLastChunks[0].SetNull(WEBAUDIO_BLOCK_SIZE);
}
}
}
void TrackUnionStream::CopyTrackData(StreamBuffer::Track* aInputTrack,
uint32_t aMapIndex, GraphTime aFrom, GraphTime aTo,
bool* aOutputTrackFinished)
{
TrackMapEntry* map = &mTrackMap[aMapIndex];
StreamBuffer::Track* outputTrack = mBuffer.FindTrack(map->mOutputTrackID);
MOZ_ASSERT(outputTrack && !outputTrack->IsEnded(), "Can't copy to ended track");
MediaSegment* segment = map->mSegment;
MediaStream* source = map->mInputPort->GetSource();
GraphTime next;
*aOutputTrackFinished = false;
for (GraphTime t = aFrom; t < aTo; t = next) {
MediaInputPort::InputInterval interval = map->mInputPort->GetNextInputInterval(t);
interval.mEnd = std::min(interval.mEnd, aTo);
StreamTime inputEnd = source->GraphTimeToStreamTime(interval.mEnd);
StreamTime inputTrackEndPoint = STREAM_TIME_MAX;
if (aInputTrack->IsEnded() &&
aInputTrack->GetEnd() <= inputEnd) {
inputTrackEndPoint = aInputTrack->GetEnd();
*aOutputTrackFinished = true;
}
if (interval.mStart >= interval.mEnd) {
break;
}
StreamTime ticks = interval.mEnd - interval.mStart;
next = interval.mEnd;
StreamTime outputStart = outputTrack->GetEnd();
if (interval.mInputIsBlocked) {
// Maybe the input track ended?
segment->AppendNullData(ticks);
STREAM_LOG(PR_LOG_DEBUG+1, ("TrackUnionStream %p appending %lld ticks of null data to track %d",
this, (long long)ticks, outputTrack->GetID()));
} else if (InMutedCycle()) {
segment->AppendNullData(ticks);
} else {
MOZ_ASSERT(outputTrack->GetEnd() == GraphTimeToStreamTime(interval.mStart),
"Samples missing");
StreamTime inputStart = source->GraphTimeToStreamTime(interval.mStart);
segment->AppendSlice(*aInputTrack->GetSegment(),
std::min(inputTrackEndPoint, inputStart),
std::min(inputTrackEndPoint, inputEnd));
}
ApplyTrackDisabling(outputTrack->GetID(), segment);
for (uint32_t j = 0; j < mListeners.Length(); ++j) {
MediaStreamListener* l = mListeners[j];
l->NotifyQueuedTrackChanges(Graph(), outputTrack->GetID(),
outputStart, 0, *segment);
}
outputTrack->GetSegment()->AppendFrom(segment);
}
}
void
AudioCaptureStream::ProcessInput(GraphTime aFrom, GraphTime aTo,
uint32_t aFlags)
{
uint32_t inputCount = mInputs.Length();
StreamBuffer::Track* track = EnsureTrack(mTrackId);
// Notify the DOM everything is in order.
if (!mTrackCreated) {
for (uint32_t i = 0; i < mListeners.Length(); i++) {
MediaStreamListener* l = mListeners[i];
AudioSegment tmp;
l->NotifyQueuedTrackChanges(
Graph(), mTrackId, 0, MediaStreamListener::TRACK_EVENT_CREATED, tmp);
l->NotifyFinishedTrackCreation(Graph());
}
mTrackCreated = true;
}
// If the captured stream is connected back to a object on the page (be it an
// HTMLMediaElement with a stream as source, or an AudioContext), a cycle
// situation occur. This can work if it's an AudioContext with at least one
// DelayNode, but the MSG will mute the whole cycle otherwise.
if (mFinished || InMutedCycle() || inputCount == 0) {
track->Get<AudioSegment>()->AppendNullData(aTo - aFrom);
} else {
// We mix down all the tracks of all inputs, to a stereo track. Everything
// is {up,down}-mixed to stereo.
mMixer.StartMixing();
AudioSegment output;
for (uint32_t i = 0; i < inputCount; i++) {
MediaStream* s = mInputs[i]->GetSource();
StreamBuffer::TrackIter tracks(s->GetStreamBuffer(), MediaSegment::AUDIO);
while (!tracks.IsEnded()) {
AudioSegment* inputSegment = tracks->Get<AudioSegment>();
StreamTime inputStart = s->GraphTimeToStreamTimeWithBlocking(aFrom);
StreamTime inputEnd = s->GraphTimeToStreamTimeWithBlocking(aTo);
AudioSegment toMix;
toMix.AppendSlice(*inputSegment, inputStart, inputEnd);
// Care for streams blocked in the [aTo, aFrom] range.
if (inputEnd - inputStart < aTo - aFrom) {
toMix.AppendNullData((aTo - aFrom) - (inputEnd - inputStart));
}
toMix.Mix(mMixer, MONO, Graph()->GraphRate());
tracks.Next();
}
}
// This calls MixerCallback below
mMixer.FinishMixing();
}
// Regardless of the status of the input tracks, we go foward.
mBuffer.AdvanceKnownTracksTime(GraphTimeToStreamTimeWithBlocking((aTo)));
}
void
AudioNodeStream::ProduceOutputBeforeInput(GraphTime aFrom)
{
MOZ_ASSERT(mEngine->AsDelayNodeEngine());
MOZ_ASSERT(mEngine->OutputCount() == 1,
"DelayNodeEngine output count should be 1");
MOZ_ASSERT(!InMutedCycle(), "DelayNodes should break cycles");
MOZ_ASSERT(mLastChunks.Length() == 1);
if (!mIsActive) {
mLastChunks[0].SetNull(WEBAUDIO_BLOCK_SIZE);
} else {
mEngine->ProduceBlockBeforeInput(this, aFrom, &mLastChunks[0]);
NS_ASSERTION(mLastChunks[0].GetDuration() == WEBAUDIO_BLOCK_SIZE,
"Invalid WebAudio chunk size");
if (mDisabledTrackIDs.Contains(static_cast<TrackID>(AUDIO_TRACK))) {
mLastChunks[0].SetNull(WEBAUDIO_BLOCK_SIZE);
}
}
}
// The MediaStreamGraph guarantees that this is actually one block, for
// AudioNodeStreams.
void
AudioNodeStream::ProcessInput(GraphTime aFrom, GraphTime aTo, uint32_t aFlags)
{
uint16_t outputCount = mLastChunks.Length();
MOZ_ASSERT(outputCount == std::max(uint16_t(1), mEngine->OutputCount()));
if (!mIsActive) {
// mLastChunks are already null.
#ifdef DEBUG
for (const auto& chunk : mLastChunks) {
MOZ_ASSERT(chunk.IsNull());
}
#endif
} else if (InMutedCycle()) {
mInputChunks.Clear();
for (uint16_t i = 0; i < outputCount; ++i) {
mLastChunks[i].SetNull(WEBAUDIO_BLOCK_SIZE);
}
} else {
// We need to generate at least one input
uint16_t maxInputs = std::max(uint16_t(1), mEngine->InputCount());
mInputChunks.SetLength(maxInputs);
for (uint16_t i = 0; i < maxInputs; ++i) {
ObtainInputBlock(mInputChunks[i], i);
}
bool finished = false;
if (mPassThrough) {
MOZ_ASSERT(outputCount == 1, "For now, we only support nodes that have one output port");
mLastChunks[0] = mInputChunks[0];
} else {
if (maxInputs <= 1 && outputCount <= 1) {
mEngine->ProcessBlock(this, aFrom,
mInputChunks[0], &mLastChunks[0], &finished);
} else {
mEngine->ProcessBlocksOnPorts(this, mInputChunks, mLastChunks, &finished);
}
}
for (uint16_t i = 0; i < outputCount; ++i) {
NS_ASSERTION(mLastChunks[i].GetDuration() == WEBAUDIO_BLOCK_SIZE,
"Invalid WebAudio chunk size");
}
if (finished) {
mMarkAsFinishedAfterThisBlock = true;
if (mIsActive) {
ScheduleCheckForInactive();
}
}
if (mDisabledTrackIDs.Contains(static_cast<TrackID>(AUDIO_TRACK))) {
for (uint32_t i = 0; i < outputCount; ++i) {
mLastChunks[i].SetNull(WEBAUDIO_BLOCK_SIZE);
}
}
}
if (!mFinished) {
// Don't output anything while finished
if (mFlags & EXTERNAL_OUTPUT) {
AdvanceOutputSegment();
}
if (mMarkAsFinishedAfterThisBlock && (aFlags & ALLOW_FINISH)) {
// This stream was finished the last time that we looked at it, and all
// of the depending streams have finished their output as well, so now
// it's time to mark this stream as finished.
if (mFlags & EXTERNAL_OUTPUT) {
FinishOutput();
}
FinishOnGraphThread();
}
}
}
// The MediaStreamGraph guarantees that this is actually one block, for
// AudioNodeStreams.
void
AudioNodeStream::ProcessInput(GraphTime aFrom, GraphTime aTo, uint32_t aFlags)
{
if (!mFinished) {
EnsureTrack(AUDIO_TRACK);
}
// No more tracks will be coming
mBuffer.AdvanceKnownTracksTime(STREAM_TIME_MAX);
uint16_t outputCount = mLastChunks.Length();
MOZ_ASSERT(outputCount == std::max(uint16_t(1), mEngine->OutputCount()));
// Consider this stream blocked if it has already finished output. Normally
// mBlocked would reflect this, but due to rounding errors our audio track may
// appear to extend slightly beyond aFrom, so we might not be blocked yet.
bool blocked = mFinished || mBlocked.GetAt(aFrom);
// If the stream has finished at this time, it will be blocked.
if (blocked || InMutedCycle()) {
for (uint16_t i = 0; i < outputCount; ++i) {
mLastChunks[i].SetNull(WEBAUDIO_BLOCK_SIZE);
}
} else {
// We need to generate at least one input
uint16_t maxInputs = std::max(uint16_t(1), mEngine->InputCount());
OutputChunks inputChunks;
inputChunks.SetLength(maxInputs);
for (uint16_t i = 0; i < maxInputs; ++i) {
ObtainInputBlock(inputChunks[i], i);
}
bool finished = false;
if (mPassThrough) {
MOZ_ASSERT(outputCount == 1, "For now, we only support nodes that have one output port");
mLastChunks[0] = inputChunks[0];
} else {
if (maxInputs <= 1 && outputCount <= 1) {
mEngine->ProcessBlock(this, inputChunks[0], &mLastChunks[0], &finished);
} else {
mEngine->ProcessBlocksOnPorts(this, inputChunks, mLastChunks, &finished);
}
}
for (uint16_t i = 0; i < outputCount; ++i) {
NS_ASSERTION(mLastChunks[i].GetDuration() == WEBAUDIO_BLOCK_SIZE,
"Invalid WebAudio chunk size");
}
if (finished) {
mMarkAsFinishedAfterThisBlock = true;
}
if (mDisabledTrackIDs.Contains(static_cast<TrackID>(AUDIO_TRACK))) {
for (uint32_t i = 0; i < outputCount; ++i) {
mLastChunks[i].SetNull(WEBAUDIO_BLOCK_SIZE);
}
}
}
if (!blocked) {
// Don't output anything while blocked
AdvanceOutputSegment();
if (mMarkAsFinishedAfterThisBlock && (aFlags & ALLOW_FINISH)) {
// This stream was finished the last time that we looked at it, and all
// of the depending streams have finished their output as well, so now
// it's time to mark this stream as finished.
FinishOutput();
}
}
}
void TrackUnionStream::CopyTrackData(StreamTracks::Track* aInputTrack,
uint32_t aMapIndex, GraphTime aFrom, GraphTime aTo,
bool* aOutputTrackFinished)
{
TrackMapEntry* map = &mTrackMap[aMapIndex];
StreamTracks::Track* outputTrack = mTracks.FindTrack(map->mOutputTrackID);
MOZ_ASSERT(outputTrack && !outputTrack->IsEnded(), "Can't copy to ended track");
MediaSegment* segment = map->mSegment;
MediaStream* source = map->mInputPort->GetSource();
GraphTime next;
*aOutputTrackFinished = false;
for (GraphTime t = aFrom; t < aTo; t = next) {
MediaInputPort::InputInterval interval = map->mInputPort->GetNextInputInterval(t);
interval.mEnd = std::min(interval.mEnd, aTo);
StreamTime inputEnd = source->GraphTimeToStreamTimeWithBlocking(interval.mEnd);
StreamTime inputTrackEndPoint = STREAM_TIME_MAX;
if (aInputTrack->IsEnded() &&
aInputTrack->GetEnd() <= inputEnd) {
inputTrackEndPoint = aInputTrack->GetEnd();
*aOutputTrackFinished = true;
}
if (interval.mStart >= interval.mEnd) {
break;
}
StreamTime ticks = interval.mEnd - interval.mStart;
next = interval.mEnd;
StreamTime outputStart = outputTrack->GetEnd();
if (interval.mInputIsBlocked) {
// Maybe the input track ended?
segment->AppendNullData(ticks);
STREAM_LOG(LogLevel::Verbose, ("TrackUnionStream %p appending %lld ticks of null data to track %d",
this, (long long)ticks, outputTrack->GetID()));
} else if (InMutedCycle()) {
segment->AppendNullData(ticks);
} else {
if (source->IsSuspended()) {
segment->AppendNullData(aTo - aFrom);
} else {
MOZ_ASSERT(outputTrack->GetEnd() == GraphTimeToStreamTimeWithBlocking(interval.mStart),
"Samples missing");
StreamTime inputStart = source->GraphTimeToStreamTimeWithBlocking(interval.mStart);
segment->AppendSlice(*aInputTrack->GetSegment(),
std::min(inputTrackEndPoint, inputStart),
std::min(inputTrackEndPoint, inputEnd));
}
}
ApplyTrackDisabling(outputTrack->GetID(), segment);
for (uint32_t j = 0; j < mListeners.Length(); ++j) {
MediaStreamListener* l = mListeners[j];
// Separate Audio and Video.
if (segment->GetType() == MediaSegment::AUDIO) {
l->NotifyQueuedAudioData(Graph(), outputTrack->GetID(),
outputStart,
*static_cast<AudioSegment*>(segment),
map->mInputPort->GetSource(),
map->mInputTrackID);
}
}
for (TrackBound<MediaStreamTrackListener>& b : mTrackListeners) {
if (b.mTrackID != outputTrack->GetID()) {
continue;
}
b.mListener->NotifyQueuedChanges(Graph(), outputStart, *segment);
}
outputTrack->GetSegment()->AppendFrom(segment);
}
}
void TrackUnionStream::CopyTrackData(StreamTracks::Track* aInputTrack,
uint32_t aMapIndex, GraphTime aFrom,
GraphTime aTo,
bool* aOutputTrackFinished) {
TrackMapEntry* map = &mTrackMap[aMapIndex];
TRACE_AUDIO_CALLBACK_COMMENT(
"Input stream %p track %i -> TrackUnionStream %p track %i",
map->mInputPort->GetSource(), map->mInputTrackID, this,
map->mOutputTrackID);
StreamTracks::Track* outputTrack = mTracks.FindTrack(map->mOutputTrackID);
MOZ_ASSERT(outputTrack && !outputTrack->IsEnded(),
"Can't copy to ended track");
MediaSegment* segment = map->mSegment;
MediaStream* source = map->mInputPort->GetSource();
GraphTime next;
*aOutputTrackFinished = false;
for (GraphTime t = aFrom; t < aTo; t = next) {
MediaInputPort::InputInterval interval =
map->mInputPort->GetNextInputInterval(t);
interval.mEnd = std::min(interval.mEnd, aTo);
StreamTime inputEnd =
source->GraphTimeToStreamTimeWithBlocking(interval.mEnd);
if (aInputTrack->IsEnded() && aInputTrack->GetEnd() <= inputEnd) {
*aOutputTrackFinished = true;
break;
}
if (interval.mStart >= interval.mEnd) {
break;
}
StreamTime ticks = interval.mEnd - interval.mStart;
next = interval.mEnd;
StreamTime outputStart = outputTrack->GetEnd();
if (interval.mInputIsBlocked) {
segment->AppendNullData(ticks);
STREAM_LOG(
LogLevel::Verbose,
("TrackUnionStream %p appending %lld ticks of null data to track %d",
this, (long long)ticks, outputTrack->GetID()));
} else if (InMutedCycle()) {
segment->AppendNullData(ticks);
} else {
if (source->IsSuspended()) {
segment->AppendNullData(aTo - aFrom);
} else {
MOZ_ASSERT(outputTrack->GetEnd() ==
GraphTimeToStreamTimeWithBlocking(interval.mStart),
"Samples missing");
StreamTime inputStart =
source->GraphTimeToStreamTimeWithBlocking(interval.mStart);
segment->AppendSlice(*aInputTrack->GetSegment(), inputStart, inputEnd);
}
}
ApplyTrackDisabling(outputTrack->GetID(), segment);
for (TrackBound<MediaStreamTrackListener>& b : mTrackListeners) {
if (b.mTrackID != outputTrack->GetID()) {
continue;
}
b.mListener->NotifyQueuedChanges(Graph(), outputStart, *segment);
}
outputTrack->GetSegment()->AppendFrom(segment);
}
}