void TrackUnionStream::CopyTrackData(StreamBuffer::Track* aInputTrack,
uint32_t aMapIndex, GraphTime aFrom, GraphTime aTo,
bool* aOutputTrackFinished)
{
TrackMapEntry* map = &mTrackMap[aMapIndex];
StreamBuffer::Track* outputTrack = mBuffer.FindTrack(map->mOutputTrackID);
MOZ_ASSERT(outputTrack && !outputTrack->IsEnded(), "Can't copy to ended track");
MediaSegment* segment = map->mSegment;
MediaStream* source = map->mInputPort->GetSource();
GraphTime next;
*aOutputTrackFinished = false;
for (GraphTime t = aFrom; t < aTo; t = next) {
MediaInputPort::InputInterval interval = map->mInputPort->GetNextInputInterval(t);
interval.mEnd = std::min(interval.mEnd, aTo);
StreamTime inputEnd = source->GraphTimeToStreamTime(interval.mEnd);
StreamTime inputTrackEndPoint = STREAM_TIME_MAX;
if (aInputTrack->IsEnded() &&
aInputTrack->GetEnd() <= inputEnd) {
inputTrackEndPoint = aInputTrack->GetEnd();
*aOutputTrackFinished = true;
}
if (interval.mStart >= interval.mEnd) {
break;
}
StreamTime ticks = interval.mEnd - interval.mStart;
next = interval.mEnd;
StreamTime outputStart = outputTrack->GetEnd();
if (interval.mInputIsBlocked) {
// Maybe the input track ended?
segment->AppendNullData(ticks);
STREAM_LOG(PR_LOG_DEBUG+1, ("TrackUnionStream %p appending %lld ticks of null data to track %d",
this, (long long)ticks, outputTrack->GetID()));
} else if (InMutedCycle()) {
segment->AppendNullData(ticks);
} else {
MOZ_ASSERT(outputTrack->GetEnd() == GraphTimeToStreamTime(interval.mStart),
"Samples missing");
StreamTime inputStart = source->GraphTimeToStreamTime(interval.mStart);
segment->AppendSlice(*aInputTrack->GetSegment(),
std::min(inputTrackEndPoint, inputStart),
std::min(inputTrackEndPoint, inputEnd));
}
ApplyTrackDisabling(outputTrack->GetID(), segment);
for (uint32_t j = 0; j < mListeners.Length(); ++j) {
MediaStreamListener* l = mListeners[j];
l->NotifyQueuedTrackChanges(Graph(), outputTrack->GetID(),
outputStart, 0, *segment);
}
outputTrack->GetSegment()->AppendFrom(segment);
}
}
uint32_t TrackUnionStream::AddTrack(MediaInputPort* aPort, StreamBuffer::Track* aTrack,
GraphTime aFrom)
{
// Use the ID of the source track if it's not already assigned to a track,
// otherwise allocate a new unique ID.
TrackID id = aTrack->GetID();
TrackID maxTrackID = 0;
for (uint32_t i = 0; i < mTrackMap.Length(); ++i) {
TrackID outID = mTrackMap[i].mOutputTrackID;
maxTrackID = std::max(maxTrackID, outID);
}
// Note: we might have removed it here, but it might still be in the
// StreamBuffer if the TrackUnionStream sees its input stream flip from
// A to B, where both A and B have a track with the same ID
while (1) {
// search until we find one not in use here, and not in mBuffer
if (!mBuffer.FindTrack(id)) {
break;
}
id = ++maxTrackID;
}
// Round up the track start time so the track, if anything, starts a
// little later than the true time. This means we'll have enough
// samples in our input stream to go just beyond the destination time.
StreamTime outputStart = GraphTimeToStreamTime(aFrom);
nsAutoPtr<MediaSegment> segment;
segment = aTrack->GetSegment()->CreateEmptyClone();
for (uint32_t j = 0; j < mListeners.Length(); ++j) {
MediaStreamListener* l = mListeners[j];
l->NotifyQueuedTrackChanges(Graph(), id, outputStart,
MediaStreamListener::TRACK_EVENT_CREATED,
*segment);
}
segment->AppendNullData(outputStart);
StreamBuffer::Track* track =
&mBuffer.AddTrack(id, outputStart, segment.forget());
STREAM_LOG(PR_LOG_DEBUG, ("TrackUnionStream %p adding track %d for input stream %p track %d, start ticks %lld",
this, id, aPort->GetSource(), aTrack->GetID(),
(long long)outputStart));
TrackMapEntry* map = mTrackMap.AppendElement();
map->mEndOfConsumedInputTicks = 0;
map->mEndOfLastInputIntervalInInputStream = -1;
map->mEndOfLastInputIntervalInOutputStream = -1;
map->mInputPort = aPort;
map->mInputTrackID = aTrack->GetID();
map->mOutputTrackID = track->GetID();
map->mSegment = aTrack->GetSegment()->CreateEmptyClone();
return mTrackMap.Length() - 1;
}
size_t
AudioNodeExternalInputStream::GetTrackMapEntry(const StreamBuffer::Track& aTrack,
GraphTime aFrom)
{
AudioSegment* segment = aTrack.Get<AudioSegment>();
// Check the map for an existing entry corresponding to the input track.
for (size_t i = 0; i < mTrackMap.Length(); ++i) {
TrackMapEntry* map = &mTrackMap[i];
if (map->mTrackID == aTrack.GetID()) {
return i;
}
}
// Determine channel count by finding the first entry with non-silent data.
AudioSegment::ChunkIterator ci(*segment);
while (!ci.IsEnded() && ci->IsNull()) {
ci.Next();
}
if (ci.IsEnded()) {
// The track is entirely silence so far, we can ignore it for now.
return nsTArray<TrackMapEntry>::NoIndex;
}
// Create a speex resampler with the same sample rate and number of channels
// as the track.
SpeexResamplerState* resampler = nullptr;
size_t channelCount = std::min((*ci).mChannelData.Length(),
WebAudioUtils::MaxChannelCount);
if (aTrack.GetRate() != mSampleRate) {
resampler = speex_resampler_init(channelCount,
aTrack.GetRate(), mSampleRate, SPEEX_RESAMPLER_QUALITY_DEFAULT, nullptr);
speex_resampler_skip_zeros(resampler);
}
TrackMapEntry* map = mTrackMap.AppendElement();
map->mEndOfConsumedInputTicks = 0;
map->mEndOfLastInputIntervalInInputStream = -1;
map->mEndOfLastInputIntervalInOutputStream = -1;
map->mSamplesPassedToResampler =
TimeToTicksRoundUp(aTrack.GetRate(), GraphTimeToStreamTime(aFrom));
map->mResampler = resampler;
map->mResamplerChannelCount = channelCount;
map->mTrackID = aTrack.GetID();
return mTrackMap.Length() - 1;
}
void
AudioNodeExternalInputStream::ProcessInput(GraphTime aFrom, GraphTime aTo,
uint32_t aFlags)
{
// According to spec, number of outputs is always 1.
MOZ_ASSERT(mLastChunks.Length() == 1);
// GC stuff can result in our input stream being destroyed before this stream.
// Handle that.
if (!IsEnabled() || mInputs.IsEmpty() || mPassThrough) {
mLastChunks[0].SetNull(WEBAUDIO_BLOCK_SIZE);
AdvanceOutputSegment();
return;
}
MOZ_ASSERT(mInputs.Length() == 1);
MediaStream* source = mInputs[0]->GetSource();
nsAutoTArray<AudioSegment,1> audioSegments;
uint32_t inputChannels = 0;
for (StreamBuffer::TrackIter tracks(source->mBuffer, MediaSegment::AUDIO);
!tracks.IsEnded(); tracks.Next()) {
const StreamBuffer::Track& inputTrack = *tracks;
const AudioSegment& inputSegment =
*static_cast<AudioSegment*>(inputTrack.GetSegment());
if (inputSegment.IsNull()) {
continue;
}
AudioSegment& segment = *audioSegments.AppendElement();
GraphTime next;
for (GraphTime t = aFrom; t < aTo; t = next) {
MediaInputPort::InputInterval interval = mInputs[0]->GetNextInputInterval(t);
interval.mEnd = std::min(interval.mEnd, aTo);
if (interval.mStart >= interval.mEnd)
break;
next = interval.mEnd;
StreamTime outputStart = GraphTimeToStreamTime(interval.mStart);
StreamTime outputEnd = GraphTimeToStreamTime(interval.mEnd);
StreamTime ticks = outputEnd - outputStart;
if (interval.mInputIsBlocked) {
segment.AppendNullData(ticks);
} else {
StreamTime inputStart =
std::min(inputSegment.GetDuration(),
source->GraphTimeToStreamTime(interval.mStart));
StreamTime inputEnd =
std::min(inputSegment.GetDuration(),
source->GraphTimeToStreamTime(interval.mEnd));
segment.AppendSlice(inputSegment, inputStart, inputEnd);
// Pad if we're looking past the end of the track
segment.AppendNullData(ticks - (inputEnd - inputStart));
}
}
for (AudioSegment::ChunkIterator iter(segment); !iter.IsEnded(); iter.Next()) {
inputChannels = GetAudioChannelsSuperset(inputChannels, iter->ChannelCount());
}
}
uint32_t accumulateIndex = 0;
if (inputChannels) {
nsAutoTArray<float,GUESS_AUDIO_CHANNELS*WEBAUDIO_BLOCK_SIZE> downmixBuffer;
for (uint32_t i = 0; i < audioSegments.Length(); ++i) {
AudioChunk tmpChunk;
ConvertSegmentToAudioBlock(&audioSegments[i], &tmpChunk, inputChannels);
if (!tmpChunk.IsNull()) {
if (accumulateIndex == 0) {
AllocateAudioBlock(inputChannels, &mLastChunks[0]);
}
AccumulateInputChunk(accumulateIndex, tmpChunk, &mLastChunks[0], &downmixBuffer);
accumulateIndex++;
}
}
}
if (accumulateIndex == 0) {
mLastChunks[0].SetNull(WEBAUDIO_BLOCK_SIZE);
}
// Using AudioNodeStream's AdvanceOutputSegment to push the media stream graph along with null data.
AdvanceOutputSegment();
}
void TrackUnionStream::ProcessInput(GraphTime aFrom, GraphTime aTo, uint32_t aFlags)
{
if (IsFinishedOnGraphThread()) {
return;
}
nsAutoTArray<bool,8> mappedTracksFinished;
nsAutoTArray<bool,8> mappedTracksWithMatchingInputTracks;
for (uint32_t i = 0; i < mTrackMap.Length(); ++i) {
mappedTracksFinished.AppendElement(true);
mappedTracksWithMatchingInputTracks.AppendElement(false);
}
bool allFinished = !mInputs.IsEmpty();
bool allHaveCurrentData = !mInputs.IsEmpty();
for (uint32_t i = 0; i < mInputs.Length(); ++i) {
MediaStream* stream = mInputs[i]->GetSource();
if (!stream->IsFinishedOnGraphThread()) {
// XXX we really should check whether 'stream' has finished within time aTo,
// not just that it's finishing when all its queued data eventually runs
// out.
allFinished = false;
}
if (!stream->HasCurrentData()) {
allHaveCurrentData = false;
}
bool trackAdded = false;
for (StreamBuffer::TrackIter tracks(stream->GetStreamBuffer());
!tracks.IsEnded(); tracks.Next()) {
bool found = false;
for (uint32_t j = 0; j < mTrackMap.Length(); ++j) {
TrackMapEntry* map = &mTrackMap[j];
if (map->mInputPort == mInputs[i] && map->mInputTrackID == tracks->GetID()) {
bool trackFinished;
StreamBuffer::Track* outputTrack = mBuffer.FindTrack(map->mOutputTrackID);
if (!outputTrack || outputTrack->IsEnded()) {
trackFinished = true;
} else {
CopyTrackData(tracks.get(), j, aFrom, aTo, &trackFinished);
}
mappedTracksFinished[j] = trackFinished;
mappedTracksWithMatchingInputTracks[j] = true;
found = true;
break;
}
}
if (!found && (!mFilterCallback || mFilterCallback(tracks.get()))) {
bool trackFinished = false;
trackAdded = true;
uint32_t mapIndex = AddTrack(mInputs[i], tracks.get(), aFrom);
CopyTrackData(tracks.get(), mapIndex, aFrom, aTo, &trackFinished);
mappedTracksFinished.AppendElement(trackFinished);
mappedTracksWithMatchingInputTracks.AppendElement(true);
}
}
if (trackAdded) {
for (MediaStreamListener* l : mListeners) {
l->NotifyFinishedTrackCreation(Graph());
}
}
}
for (int32_t i = mTrackMap.Length() - 1; i >= 0; --i) {
if (mappedTracksFinished[i]) {
EndTrack(i);
} else {
allFinished = false;
}
if (!mappedTracksWithMatchingInputTracks[i]) {
mTrackMap.RemoveElementAt(i);
}
}
if (allFinished && mAutofinish && (aFlags & ALLOW_FINISH)) {
// All streams have finished and won't add any more tracks, and
// all our tracks have actually finished and been removed from our map,
// so we're finished now.
FinishOnGraphThread();
} else {
mBuffer.AdvanceKnownTracksTime(GraphTimeToStreamTime(aTo));
}
if (allHaveCurrentData) {
// We can make progress if we're not blocked
mHasCurrentData = true;
}
}
void
AudioNodeExternalInputStream::ProcessInput(GraphTime aFrom, GraphTime aTo,
uint32_t aFlags)
{
// According to spec, number of outputs is always 1.
MOZ_ASSERT(mLastChunks.Length() == 1);
// GC stuff can result in our input stream being destroyed before this stream.
// Handle that.
if (!IsEnabled() || mInputs.IsEmpty() || mPassThrough) {
mLastChunks[0].SetNull(WEBAUDIO_BLOCK_SIZE);
return;
}
MOZ_ASSERT(mInputs.Length() == 1);
MediaStream* source = mInputs[0]->GetSource();
AutoTArray<AudioSegment,1> audioSegments;
uint32_t inputChannels = 0;
for (StreamTracks::TrackIter tracks(source->mTracks);
!tracks.IsEnded(); tracks.Next()) {
const StreamTracks::Track& inputTrack = *tracks;
if (!mInputs[0]->PassTrackThrough(tracks->GetID())) {
continue;
}
if (inputTrack.GetSegment()->GetType() == MediaSegment::VIDEO) {
MOZ_ASSERT(false, "AudioNodeExternalInputStream shouldn't have video tracks");
continue;
}
const AudioSegment& inputSegment =
*static_cast<AudioSegment*>(inputTrack.GetSegment());
if (inputSegment.IsNull()) {
continue;
}
AudioSegment& segment = *audioSegments.AppendElement();
GraphTime next;
for (GraphTime t = aFrom; t < aTo; t = next) {
MediaInputPort::InputInterval interval = mInputs[0]->GetNextInputInterval(t);
interval.mEnd = std::min(interval.mEnd, aTo);
if (interval.mStart >= interval.mEnd)
break;
next = interval.mEnd;
// We know this stream does not block during the processing interval ---
// we're not finished, we don't underrun, and we're not suspended.
StreamTime outputStart = GraphTimeToStreamTime(interval.mStart);
StreamTime outputEnd = GraphTimeToStreamTime(interval.mEnd);
StreamTime ticks = outputEnd - outputStart;
if (interval.mInputIsBlocked) {
segment.AppendNullData(ticks);
} else {
// The input stream is not blocked in this interval, so no need to call
// GraphTimeToStreamTimeWithBlocking.
StreamTime inputStart =
std::min(inputSegment.GetDuration(),
source->GraphTimeToStreamTime(interval.mStart));
StreamTime inputEnd =
std::min(inputSegment.GetDuration(),
source->GraphTimeToStreamTime(interval.mEnd));
segment.AppendSlice(inputSegment, inputStart, inputEnd);
// Pad if we're looking past the end of the track
segment.AppendNullData(ticks - (inputEnd - inputStart));
}
}
for (AudioSegment::ChunkIterator iter(segment); !iter.IsEnded(); iter.Next()) {
inputChannels = GetAudioChannelsSuperset(inputChannels, iter->ChannelCount());
}
}
uint32_t accumulateIndex = 0;
if (inputChannels) {
DownmixBufferType downmixBuffer;
ASSERT_ALIGNED16(downmixBuffer.Elements());
for (uint32_t i = 0; i < audioSegments.Length(); ++i) {
AudioBlock tmpChunk;
ConvertSegmentToAudioBlock(&audioSegments[i], &tmpChunk, inputChannels);
if (!tmpChunk.IsNull()) {
if (accumulateIndex == 0) {
mLastChunks[0].AllocateChannels(inputChannels);
}
AccumulateInputChunk(accumulateIndex, tmpChunk, &mLastChunks[0], &downmixBuffer);
accumulateIndex++;
}
}
}
if (accumulateIndex == 0) {
mLastChunks[0].SetNull(WEBAUDIO_BLOCK_SIZE);
}
}
void
AudioNodeExternalInputStream::ProcessInput(GraphTime aFrom, GraphTime aTo,
uint32_t aFlags)
{
// According to spec, number of outputs is always 1.
mLastChunks.SetLength(1);
// GC stuff can result in our input stream being destroyed before this stream.
// Handle that.
if (mInputs.IsEmpty()) {
mLastChunks[0].SetNull(WEBAUDIO_BLOCK_SIZE);
AdvanceOutputSegment();
return;
}
MOZ_ASSERT(mInputs.Length() == 1);
MediaStream* source = mInputs[0]->GetSource();
nsAutoTArray<AudioSegment,1> audioSegments;
nsAutoTArray<bool,1> trackMapEntriesUsed;
uint32_t inputChannels = 0;
for (StreamBuffer::TrackIter tracks(source->mBuffer, MediaSegment::AUDIO);
!tracks.IsEnded(); tracks.Next()) {
const StreamBuffer::Track& inputTrack = *tracks;
// Create a TrackMapEntry if necessary.
size_t trackMapIndex = GetTrackMapEntry(inputTrack, aFrom);
// Maybe there's nothing in this track yet. If so, ignore it. (While the
// track is only playing silence, we may not be able to determine the
// correct number of channels to start resampling.)
if (trackMapIndex == nsTArray<TrackMapEntry>::NoIndex) {
continue;
}
while (trackMapEntriesUsed.Length() <= trackMapIndex) {
trackMapEntriesUsed.AppendElement(false);
}
trackMapEntriesUsed[trackMapIndex] = true;
TrackMapEntry* trackMap = &mTrackMap[trackMapIndex];
AudioSegment segment;
GraphTime next;
TrackRate inputTrackRate = inputTrack.GetRate();
for (GraphTime t = aFrom; t < aTo; t = next) {
MediaInputPort::InputInterval interval = mInputs[0]->GetNextInputInterval(t);
interval.mEnd = std::min(interval.mEnd, aTo);
if (interval.mStart >= interval.mEnd)
break;
next = interval.mEnd;
// Ticks >= startTicks and < endTicks are in the interval
StreamTime outputEnd = GraphTimeToStreamTime(interval.mEnd);
TrackTicks startTicks = trackMap->mSamplesPassedToResampler + segment.GetDuration();
StreamTime outputStart = GraphTimeToStreamTime(interval.mStart);
NS_ASSERTION(startTicks == TimeToTicksRoundUp(inputTrackRate, outputStart),
"Samples missing");
TrackTicks endTicks = TimeToTicksRoundUp(inputTrackRate, outputEnd);
TrackTicks ticks = endTicks - startTicks;
if (interval.mInputIsBlocked) {
segment.AppendNullData(ticks);
} else {
// See comments in TrackUnionStream::CopyTrackData
StreamTime inputStart = source->GraphTimeToStreamTime(interval.mStart);
StreamTime inputEnd = source->GraphTimeToStreamTime(interval.mEnd);
TrackTicks inputTrackEndPoint =
inputTrack.IsEnded() ? inputTrack.GetEnd() : TRACK_TICKS_MAX;
if (trackMap->mEndOfLastInputIntervalInInputStream != inputStart ||
trackMap->mEndOfLastInputIntervalInOutputStream != outputStart) {
// Start of a new series of intervals where neither stream is blocked.
trackMap->mEndOfConsumedInputTicks = TimeToTicksRoundDown(inputTrackRate, inputStart) - 1;
}
TrackTicks inputStartTicks = trackMap->mEndOfConsumedInputTicks;
TrackTicks inputEndTicks = inputStartTicks + ticks;
trackMap->mEndOfConsumedInputTicks = inputEndTicks;
trackMap->mEndOfLastInputIntervalInInputStream = inputEnd;
trackMap->mEndOfLastInputIntervalInOutputStream = outputEnd;
if (inputStartTicks < 0) {
// Data before the start of the track is just null.
segment.AppendNullData(-inputStartTicks);
inputStartTicks = 0;
}
if (inputEndTicks > inputStartTicks) {
segment.AppendSlice(*inputTrack.GetSegment(),
std::min(inputTrackEndPoint, inputStartTicks),
std::min(inputTrackEndPoint, inputEndTicks));
}
// Pad if we're looking past the end of the track
segment.AppendNullData(ticks - segment.GetDuration());
}
}
trackMap->mSamplesPassedToResampler += segment.GetDuration();
trackMap->ResampleInputData(&segment);
if (trackMap->mResampledData.GetDuration() < mCurrentOutputPosition + WEBAUDIO_BLOCK_SIZE) {
// We don't have enough data. Delay it.
trackMap->mResampledData.InsertNullDataAtStart(
mCurrentOutputPosition + WEBAUDIO_BLOCK_SIZE - trackMap->mResampledData.GetDuration());
}
audioSegments.AppendElement()->AppendSlice(trackMap->mResampledData,
mCurrentOutputPosition, mCurrentOutputPosition + WEBAUDIO_BLOCK_SIZE);
trackMap->mResampledData.ForgetUpTo(mCurrentOutputPosition + WEBAUDIO_BLOCK_SIZE);
inputChannels = GetAudioChannelsSuperset(inputChannels, trackMap->mResamplerChannelCount);
}
for (int32_t i = mTrackMap.Length() - 1; i >= 0; --i) {
if (i >= int32_t(trackMapEntriesUsed.Length()) || !trackMapEntriesUsed[i]) {
mTrackMap.RemoveElementAt(i);
}
}
uint32_t accumulateIndex = 0;
if (inputChannels) {
nsAutoTArray<float,GUESS_AUDIO_CHANNELS*WEBAUDIO_BLOCK_SIZE> downmixBuffer;
for (uint32_t i = 0; i < audioSegments.Length(); ++i) {
AudioChunk tmpChunk;
ConvertSegmentToAudioBlock(&audioSegments[i], &tmpChunk);
if (!tmpChunk.IsNull()) {
if (accumulateIndex == 0) {
AllocateAudioBlock(inputChannels, &mLastChunks[0]);
}
AccumulateInputChunk(accumulateIndex, tmpChunk, &mLastChunks[0], &downmixBuffer);
accumulateIndex++;
}
}
}
if (accumulateIndex == 0) {
mLastChunks[0].SetNull(WEBAUDIO_BLOCK_SIZE);
}
mCurrentOutputPosition += WEBAUDIO_BLOCK_SIZE;
// Using AudioNodeStream's AdvanceOutputSegment to push the media stream graph along with null data.
AdvanceOutputSegment();
}
