void TrackUnionStream::CopyTrackData(StreamBuffer::Track* aInputTrack, uint32_t aMapIndex, GraphTime aFrom, GraphTime aTo, bool* aOutputTrackFinished) { TrackMapEntry* map = &mTrackMap[aMapIndex]; StreamBuffer::Track* outputTrack = mBuffer.FindTrack(map->mOutputTrackID); MOZ_ASSERT(outputTrack && !outputTrack->IsEnded(), "Can't copy to ended track"); MediaSegment* segment = map->mSegment; MediaStream* source = map->mInputPort->GetSource(); GraphTime next; *aOutputTrackFinished = false; for (GraphTime t = aFrom; t < aTo; t = next) { MediaInputPort::InputInterval interval = map->mInputPort->GetNextInputInterval(t); interval.mEnd = std::min(interval.mEnd, aTo); StreamTime inputEnd = source->GraphTimeToStreamTime(interval.mEnd); StreamTime inputTrackEndPoint = STREAM_TIME_MAX; if (aInputTrack->IsEnded() && aInputTrack->GetEnd() <= inputEnd) { inputTrackEndPoint = aInputTrack->GetEnd(); *aOutputTrackFinished = true; } if (interval.mStart >= interval.mEnd) { break; } StreamTime ticks = interval.mEnd - interval.mStart; next = interval.mEnd; StreamTime outputStart = outputTrack->GetEnd(); if (interval.mInputIsBlocked) { // Maybe the input track ended? segment->AppendNullData(ticks); STREAM_LOG(PR_LOG_DEBUG+1, ("TrackUnionStream %p appending %lld ticks of null data to track %d", this, (long long)ticks, outputTrack->GetID())); } else if (InMutedCycle()) { segment->AppendNullData(ticks); } else { MOZ_ASSERT(outputTrack->GetEnd() == GraphTimeToStreamTime(interval.mStart), "Samples missing"); StreamTime inputStart = source->GraphTimeToStreamTime(interval.mStart); segment->AppendSlice(*aInputTrack->GetSegment(), std::min(inputTrackEndPoint, inputStart), std::min(inputTrackEndPoint, inputEnd)); } ApplyTrackDisabling(outputTrack->GetID(), segment); for (uint32_t j = 0; j < mListeners.Length(); ++j) { MediaStreamListener* l = mListeners[j]; l->NotifyQueuedTrackChanges(Graph(), outputTrack->GetID(), outputStart, 0, *segment); } outputTrack->GetSegment()->AppendFrom(segment); } }
uint32_t TrackUnionStream::AddTrack(MediaInputPort* aPort, StreamBuffer::Track* aTrack, GraphTime aFrom) { // Use the ID of the source track if it's not already assigned to a track, // otherwise allocate a new unique ID. TrackID id = aTrack->GetID(); TrackID maxTrackID = 0; for (uint32_t i = 0; i < mTrackMap.Length(); ++i) { TrackID outID = mTrackMap[i].mOutputTrackID; maxTrackID = std::max(maxTrackID, outID); } // Note: we might have removed it here, but it might still be in the // StreamBuffer if the TrackUnionStream sees its input stream flip from // A to B, where both A and B have a track with the same ID while (1) { // search until we find one not in use here, and not in mBuffer if (!mBuffer.FindTrack(id)) { break; } id = ++maxTrackID; } // Round up the track start time so the track, if anything, starts a // little later than the true time. This means we'll have enough // samples in our input stream to go just beyond the destination time. StreamTime outputStart = GraphTimeToStreamTime(aFrom); nsAutoPtr<MediaSegment> segment; segment = aTrack->GetSegment()->CreateEmptyClone(); for (uint32_t j = 0; j < mListeners.Length(); ++j) { MediaStreamListener* l = mListeners[j]; l->NotifyQueuedTrackChanges(Graph(), id, outputStart, MediaStreamListener::TRACK_EVENT_CREATED, *segment); } segment->AppendNullData(outputStart); StreamBuffer::Track* track = &mBuffer.AddTrack(id, outputStart, segment.forget()); STREAM_LOG(PR_LOG_DEBUG, ("TrackUnionStream %p adding track %d for input stream %p track %d, start ticks %lld", this, id, aPort->GetSource(), aTrack->GetID(), (long long)outputStart)); TrackMapEntry* map = mTrackMap.AppendElement(); map->mEndOfConsumedInputTicks = 0; map->mEndOfLastInputIntervalInInputStream = -1; map->mEndOfLastInputIntervalInOutputStream = -1; map->mInputPort = aPort; map->mInputTrackID = aTrack->GetID(); map->mOutputTrackID = track->GetID(); map->mSegment = aTrack->GetSegment()->CreateEmptyClone(); return mTrackMap.Length() - 1; }
size_t AudioNodeExternalInputStream::GetTrackMapEntry(const StreamBuffer::Track& aTrack, GraphTime aFrom) { AudioSegment* segment = aTrack.Get<AudioSegment>(); // Check the map for an existing entry corresponding to the input track. for (size_t i = 0; i < mTrackMap.Length(); ++i) { TrackMapEntry* map = &mTrackMap[i]; if (map->mTrackID == aTrack.GetID()) { return i; } } // Determine channel count by finding the first entry with non-silent data. AudioSegment::ChunkIterator ci(*segment); while (!ci.IsEnded() && ci->IsNull()) { ci.Next(); } if (ci.IsEnded()) { // The track is entirely silence so far, we can ignore it for now. return nsTArray<TrackMapEntry>::NoIndex; } // Create a speex resampler with the same sample rate and number of channels // as the track. SpeexResamplerState* resampler = nullptr; size_t channelCount = std::min((*ci).mChannelData.Length(), WebAudioUtils::MaxChannelCount); if (aTrack.GetRate() != mSampleRate) { resampler = speex_resampler_init(channelCount, aTrack.GetRate(), mSampleRate, SPEEX_RESAMPLER_QUALITY_DEFAULT, nullptr); speex_resampler_skip_zeros(resampler); } TrackMapEntry* map = mTrackMap.AppendElement(); map->mEndOfConsumedInputTicks = 0; map->mEndOfLastInputIntervalInInputStream = -1; map->mEndOfLastInputIntervalInOutputStream = -1; map->mSamplesPassedToResampler = TimeToTicksRoundUp(aTrack.GetRate(), GraphTimeToStreamTime(aFrom)); map->mResampler = resampler; map->mResamplerChannelCount = channelCount; map->mTrackID = aTrack.GetID(); return mTrackMap.Length() - 1; }
void AudioNodeExternalInputStream::ProcessInput(GraphTime aFrom, GraphTime aTo, uint32_t aFlags) { // According to spec, number of outputs is always 1. MOZ_ASSERT(mLastChunks.Length() == 1); // GC stuff can result in our input stream being destroyed before this stream. // Handle that. if (!IsEnabled() || mInputs.IsEmpty() || mPassThrough) { mLastChunks[0].SetNull(WEBAUDIO_BLOCK_SIZE); AdvanceOutputSegment(); return; } MOZ_ASSERT(mInputs.Length() == 1); MediaStream* source = mInputs[0]->GetSource(); nsAutoTArray<AudioSegment,1> audioSegments; uint32_t inputChannels = 0; for (StreamBuffer::TrackIter tracks(source->mBuffer, MediaSegment::AUDIO); !tracks.IsEnded(); tracks.Next()) { const StreamBuffer::Track& inputTrack = *tracks; const AudioSegment& inputSegment = *static_cast<AudioSegment*>(inputTrack.GetSegment()); if (inputSegment.IsNull()) { continue; } AudioSegment& segment = *audioSegments.AppendElement(); GraphTime next; for (GraphTime t = aFrom; t < aTo; t = next) { MediaInputPort::InputInterval interval = mInputs[0]->GetNextInputInterval(t); interval.mEnd = std::min(interval.mEnd, aTo); if (interval.mStart >= interval.mEnd) break; next = interval.mEnd; StreamTime outputStart = GraphTimeToStreamTime(interval.mStart); StreamTime outputEnd = GraphTimeToStreamTime(interval.mEnd); StreamTime ticks = outputEnd - outputStart; if (interval.mInputIsBlocked) { segment.AppendNullData(ticks); } else { StreamTime inputStart = std::min(inputSegment.GetDuration(), source->GraphTimeToStreamTime(interval.mStart)); StreamTime inputEnd = std::min(inputSegment.GetDuration(), source->GraphTimeToStreamTime(interval.mEnd)); segment.AppendSlice(inputSegment, inputStart, inputEnd); // Pad if we're looking past the end of the track segment.AppendNullData(ticks - (inputEnd - inputStart)); } } for (AudioSegment::ChunkIterator iter(segment); !iter.IsEnded(); iter.Next()) { inputChannels = GetAudioChannelsSuperset(inputChannels, iter->ChannelCount()); } } uint32_t accumulateIndex = 0; if (inputChannels) { nsAutoTArray<float,GUESS_AUDIO_CHANNELS*WEBAUDIO_BLOCK_SIZE> downmixBuffer; for (uint32_t i = 0; i < audioSegments.Length(); ++i) { AudioChunk tmpChunk; ConvertSegmentToAudioBlock(&audioSegments[i], &tmpChunk, inputChannels); if (!tmpChunk.IsNull()) { if (accumulateIndex == 0) { AllocateAudioBlock(inputChannels, &mLastChunks[0]); } AccumulateInputChunk(accumulateIndex, tmpChunk, &mLastChunks[0], &downmixBuffer); accumulateIndex++; } } } if (accumulateIndex == 0) { mLastChunks[0].SetNull(WEBAUDIO_BLOCK_SIZE); } // Using AudioNodeStream's AdvanceOutputSegment to push the media stream graph along with null data. AdvanceOutputSegment(); }
void TrackUnionStream::ProcessInput(GraphTime aFrom, GraphTime aTo, uint32_t aFlags) { if (IsFinishedOnGraphThread()) { return; } nsAutoTArray<bool,8> mappedTracksFinished; nsAutoTArray<bool,8> mappedTracksWithMatchingInputTracks; for (uint32_t i = 0; i < mTrackMap.Length(); ++i) { mappedTracksFinished.AppendElement(true); mappedTracksWithMatchingInputTracks.AppendElement(false); } bool allFinished = !mInputs.IsEmpty(); bool allHaveCurrentData = !mInputs.IsEmpty(); for (uint32_t i = 0; i < mInputs.Length(); ++i) { MediaStream* stream = mInputs[i]->GetSource(); if (!stream->IsFinishedOnGraphThread()) { // XXX we really should check whether 'stream' has finished within time aTo, // not just that it's finishing when all its queued data eventually runs // out. allFinished = false; } if (!stream->HasCurrentData()) { allHaveCurrentData = false; } bool trackAdded = false; for (StreamBuffer::TrackIter tracks(stream->GetStreamBuffer()); !tracks.IsEnded(); tracks.Next()) { bool found = false; for (uint32_t j = 0; j < mTrackMap.Length(); ++j) { TrackMapEntry* map = &mTrackMap[j]; if (map->mInputPort == mInputs[i] && map->mInputTrackID == tracks->GetID()) { bool trackFinished; StreamBuffer::Track* outputTrack = mBuffer.FindTrack(map->mOutputTrackID); if (!outputTrack || outputTrack->IsEnded()) { trackFinished = true; } else { CopyTrackData(tracks.get(), j, aFrom, aTo, &trackFinished); } mappedTracksFinished[j] = trackFinished; mappedTracksWithMatchingInputTracks[j] = true; found = true; break; } } if (!found && (!mFilterCallback || mFilterCallback(tracks.get()))) { bool trackFinished = false; trackAdded = true; uint32_t mapIndex = AddTrack(mInputs[i], tracks.get(), aFrom); CopyTrackData(tracks.get(), mapIndex, aFrom, aTo, &trackFinished); mappedTracksFinished.AppendElement(trackFinished); mappedTracksWithMatchingInputTracks.AppendElement(true); } } if (trackAdded) { for (MediaStreamListener* l : mListeners) { l->NotifyFinishedTrackCreation(Graph()); } } } for (int32_t i = mTrackMap.Length() - 1; i >= 0; --i) { if (mappedTracksFinished[i]) { EndTrack(i); } else { allFinished = false; } if (!mappedTracksWithMatchingInputTracks[i]) { mTrackMap.RemoveElementAt(i); } } if (allFinished && mAutofinish && (aFlags & ALLOW_FINISH)) { // All streams have finished and won't add any more tracks, and // all our tracks have actually finished and been removed from our map, // so we're finished now. FinishOnGraphThread(); } else { mBuffer.AdvanceKnownTracksTime(GraphTimeToStreamTime(aTo)); } if (allHaveCurrentData) { // We can make progress if we're not blocked mHasCurrentData = true; } }
void AudioNodeExternalInputStream::ProcessInput(GraphTime aFrom, GraphTime aTo, uint32_t aFlags) { // According to spec, number of outputs is always 1. MOZ_ASSERT(mLastChunks.Length() == 1); // GC stuff can result in our input stream being destroyed before this stream. // Handle that. if (!IsEnabled() || mInputs.IsEmpty() || mPassThrough) { mLastChunks[0].SetNull(WEBAUDIO_BLOCK_SIZE); return; } MOZ_ASSERT(mInputs.Length() == 1); MediaStream* source = mInputs[0]->GetSource(); AutoTArray<AudioSegment,1> audioSegments; uint32_t inputChannels = 0; for (StreamTracks::TrackIter tracks(source->mTracks); !tracks.IsEnded(); tracks.Next()) { const StreamTracks::Track& inputTrack = *tracks; if (!mInputs[0]->PassTrackThrough(tracks->GetID())) { continue; } if (inputTrack.GetSegment()->GetType() == MediaSegment::VIDEO) { MOZ_ASSERT(false, "AudioNodeExternalInputStream shouldn't have video tracks"); continue; } const AudioSegment& inputSegment = *static_cast<AudioSegment*>(inputTrack.GetSegment()); if (inputSegment.IsNull()) { continue; } AudioSegment& segment = *audioSegments.AppendElement(); GraphTime next; for (GraphTime t = aFrom; t < aTo; t = next) { MediaInputPort::InputInterval interval = mInputs[0]->GetNextInputInterval(t); interval.mEnd = std::min(interval.mEnd, aTo); if (interval.mStart >= interval.mEnd) break; next = interval.mEnd; // We know this stream does not block during the processing interval --- // we're not finished, we don't underrun, and we're not suspended. StreamTime outputStart = GraphTimeToStreamTime(interval.mStart); StreamTime outputEnd = GraphTimeToStreamTime(interval.mEnd); StreamTime ticks = outputEnd - outputStart; if (interval.mInputIsBlocked) { segment.AppendNullData(ticks); } else { // The input stream is not blocked in this interval, so no need to call // GraphTimeToStreamTimeWithBlocking. StreamTime inputStart = std::min(inputSegment.GetDuration(), source->GraphTimeToStreamTime(interval.mStart)); StreamTime inputEnd = std::min(inputSegment.GetDuration(), source->GraphTimeToStreamTime(interval.mEnd)); segment.AppendSlice(inputSegment, inputStart, inputEnd); // Pad if we're looking past the end of the track segment.AppendNullData(ticks - (inputEnd - inputStart)); } } for (AudioSegment::ChunkIterator iter(segment); !iter.IsEnded(); iter.Next()) { inputChannels = GetAudioChannelsSuperset(inputChannels, iter->ChannelCount()); } } uint32_t accumulateIndex = 0; if (inputChannels) { DownmixBufferType downmixBuffer; ASSERT_ALIGNED16(downmixBuffer.Elements()); for (uint32_t i = 0; i < audioSegments.Length(); ++i) { AudioBlock tmpChunk; ConvertSegmentToAudioBlock(&audioSegments[i], &tmpChunk, inputChannels); if (!tmpChunk.IsNull()) { if (accumulateIndex == 0) { mLastChunks[0].AllocateChannels(inputChannels); } AccumulateInputChunk(accumulateIndex, tmpChunk, &mLastChunks[0], &downmixBuffer); accumulateIndex++; } } } if (accumulateIndex == 0) { mLastChunks[0].SetNull(WEBAUDIO_BLOCK_SIZE); } }
void AudioNodeExternalInputStream::ProcessInput(GraphTime aFrom, GraphTime aTo, uint32_t aFlags) { // According to spec, number of outputs is always 1. mLastChunks.SetLength(1); // GC stuff can result in our input stream being destroyed before this stream. // Handle that. if (mInputs.IsEmpty()) { mLastChunks[0].SetNull(WEBAUDIO_BLOCK_SIZE); AdvanceOutputSegment(); return; } MOZ_ASSERT(mInputs.Length() == 1); MediaStream* source = mInputs[0]->GetSource(); nsAutoTArray<AudioSegment,1> audioSegments; nsAutoTArray<bool,1> trackMapEntriesUsed; uint32_t inputChannels = 0; for (StreamBuffer::TrackIter tracks(source->mBuffer, MediaSegment::AUDIO); !tracks.IsEnded(); tracks.Next()) { const StreamBuffer::Track& inputTrack = *tracks; // Create a TrackMapEntry if necessary. size_t trackMapIndex = GetTrackMapEntry(inputTrack, aFrom); // Maybe there's nothing in this track yet. If so, ignore it. (While the // track is only playing silence, we may not be able to determine the // correct number of channels to start resampling.) if (trackMapIndex == nsTArray<TrackMapEntry>::NoIndex) { continue; } while (trackMapEntriesUsed.Length() <= trackMapIndex) { trackMapEntriesUsed.AppendElement(false); } trackMapEntriesUsed[trackMapIndex] = true; TrackMapEntry* trackMap = &mTrackMap[trackMapIndex]; AudioSegment segment; GraphTime next; TrackRate inputTrackRate = inputTrack.GetRate(); for (GraphTime t = aFrom; t < aTo; t = next) { MediaInputPort::InputInterval interval = mInputs[0]->GetNextInputInterval(t); interval.mEnd = std::min(interval.mEnd, aTo); if (interval.mStart >= interval.mEnd) break; next = interval.mEnd; // Ticks >= startTicks and < endTicks are in the interval StreamTime outputEnd = GraphTimeToStreamTime(interval.mEnd); TrackTicks startTicks = trackMap->mSamplesPassedToResampler + segment.GetDuration(); StreamTime outputStart = GraphTimeToStreamTime(interval.mStart); NS_ASSERTION(startTicks == TimeToTicksRoundUp(inputTrackRate, outputStart), "Samples missing"); TrackTicks endTicks = TimeToTicksRoundUp(inputTrackRate, outputEnd); TrackTicks ticks = endTicks - startTicks; if (interval.mInputIsBlocked) { segment.AppendNullData(ticks); } else { // See comments in TrackUnionStream::CopyTrackData StreamTime inputStart = source->GraphTimeToStreamTime(interval.mStart); StreamTime inputEnd = source->GraphTimeToStreamTime(interval.mEnd); TrackTicks inputTrackEndPoint = inputTrack.IsEnded() ? inputTrack.GetEnd() : TRACK_TICKS_MAX; if (trackMap->mEndOfLastInputIntervalInInputStream != inputStart || trackMap->mEndOfLastInputIntervalInOutputStream != outputStart) { // Start of a new series of intervals where neither stream is blocked. trackMap->mEndOfConsumedInputTicks = TimeToTicksRoundDown(inputTrackRate, inputStart) - 1; } TrackTicks inputStartTicks = trackMap->mEndOfConsumedInputTicks; TrackTicks inputEndTicks = inputStartTicks + ticks; trackMap->mEndOfConsumedInputTicks = inputEndTicks; trackMap->mEndOfLastInputIntervalInInputStream = inputEnd; trackMap->mEndOfLastInputIntervalInOutputStream = outputEnd; if (inputStartTicks < 0) { // Data before the start of the track is just null. segment.AppendNullData(-inputStartTicks); inputStartTicks = 0; } if (inputEndTicks > inputStartTicks) { segment.AppendSlice(*inputTrack.GetSegment(), std::min(inputTrackEndPoint, inputStartTicks), std::min(inputTrackEndPoint, inputEndTicks)); } // Pad if we're looking past the end of the track segment.AppendNullData(ticks - segment.GetDuration()); } } trackMap->mSamplesPassedToResampler += segment.GetDuration(); trackMap->ResampleInputData(&segment); if (trackMap->mResampledData.GetDuration() < mCurrentOutputPosition + WEBAUDIO_BLOCK_SIZE) { // We don't have enough data. Delay it. trackMap->mResampledData.InsertNullDataAtStart( mCurrentOutputPosition + WEBAUDIO_BLOCK_SIZE - trackMap->mResampledData.GetDuration()); } audioSegments.AppendElement()->AppendSlice(trackMap->mResampledData, mCurrentOutputPosition, mCurrentOutputPosition + WEBAUDIO_BLOCK_SIZE); trackMap->mResampledData.ForgetUpTo(mCurrentOutputPosition + WEBAUDIO_BLOCK_SIZE); inputChannels = GetAudioChannelsSuperset(inputChannels, trackMap->mResamplerChannelCount); } for (int32_t i = mTrackMap.Length() - 1; i >= 0; --i) { if (i >= int32_t(trackMapEntriesUsed.Length()) || !trackMapEntriesUsed[i]) { mTrackMap.RemoveElementAt(i); } } uint32_t accumulateIndex = 0; if (inputChannels) { nsAutoTArray<float,GUESS_AUDIO_CHANNELS*WEBAUDIO_BLOCK_SIZE> downmixBuffer; for (uint32_t i = 0; i < audioSegments.Length(); ++i) { AudioChunk tmpChunk; ConvertSegmentToAudioBlock(&audioSegments[i], &tmpChunk); if (!tmpChunk.IsNull()) { if (accumulateIndex == 0) { AllocateAudioBlock(inputChannels, &mLastChunks[0]); } AccumulateInputChunk(accumulateIndex, tmpChunk, &mLastChunks[0], &downmixBuffer); accumulateIndex++; } } } if (accumulateIndex == 0) { mLastChunks[0].SetNull(WEBAUDIO_BLOCK_SIZE); } mCurrentOutputPosition += WEBAUDIO_BLOCK_SIZE; // Using AudioNodeStream's AdvanceOutputSegment to push the media stream graph along with null data. AdvanceOutputSegment(); }