// The MediaStreamGraph guarantees that this is actually one block, for // AudioNodeStreams. void AudioNodeStream::ProcessInput(GraphTime aFrom, GraphTime aTo, uint32_t aFlags) { EnsureTrack(AUDIO_TRACK, mSampleRate); // No more tracks will be coming mBuffer.AdvanceKnownTracksTime(STREAM_TIME_MAX); uint16_t outputCount = std::max(uint16_t(1), mEngine->OutputCount()); mLastChunks.SetLength(outputCount); // Consider this stream blocked if it has already finished output. Normally // mBlocked would reflect this, but due to rounding errors our audio track may // appear to extend slightly beyond aFrom, so we might not be blocked yet. bool blocked = mFinished || mBlocked.GetAt(aFrom); // If the stream has finished at this time, it will be blocked. if (mMuted || blocked) { for (uint16_t i = 0; i < outputCount; ++i) { mLastChunks[i].SetNull(WEBAUDIO_BLOCK_SIZE); } } else { // We need to generate at least one input uint16_t maxInputs = std::max(uint16_t(1), mEngine->InputCount()); OutputChunks inputChunks; inputChunks.SetLength(maxInputs); for (uint16_t i = 0; i < maxInputs; ++i) { ObtainInputBlock(inputChunks[i], i); } bool finished = false; if (maxInputs <= 1 && mEngine->OutputCount() <= 1) { mEngine->ProcessBlock(this, inputChunks[0], &mLastChunks[0], &finished); } else { mEngine->ProcessBlocksOnPorts(this, inputChunks, mLastChunks, &finished); } for (uint16_t i = 0; i < outputCount; ++i) { NS_ASSERTION(mLastChunks[i].GetDuration() == WEBAUDIO_BLOCK_SIZE, "Invalid WebAudio chunk size"); } if (finished) { mMarkAsFinishedAfterThisBlock = true; } if (mDisabledTrackIDs.Contains(static_cast<TrackID>(AUDIO_TRACK))) { for (uint32_t i = 0; i < outputCount; ++i) { mLastChunks[i].SetNull(WEBAUDIO_BLOCK_SIZE); } } } if (!blocked) { // Don't output anything while blocked AdvanceOutputSegment(); if (mMarkAsFinishedAfterThisBlock && (aFlags & ALLOW_FINISH)) { // This stream was finished the last time that we looked at it, and all // of the depending streams have finished their output as well, so now // it's time to mark this stream as finished. FinishOutput(); } } }
// The MediaStreamGraph guarantees that this is actually one block, for // AudioNodeStreams. void AudioNodeStream::ProduceOutput(GraphTime aFrom, GraphTime aTo) { if (mMarkAsFinishedAfterThisBlock) { // This stream was finished the last time that we looked at it, and all // of the depending streams have finished their output as well, so now // it's time to mark this stream as finished. FinishOutput(); } EnsureTrack(AUDIO_NODE_STREAM_TRACK_ID, mSampleRate); uint16_t outputCount = std::max(uint16_t(1), mEngine->OutputCount()); mLastChunks.SetLength(outputCount); if (mInCycle) { // XXX DelayNode not supported yet so just produce silence for (uint16_t i = 0; i < outputCount; ++i) { mLastChunks[i].SetNull(WEBAUDIO_BLOCK_SIZE); } } else { for (uint16_t i = 0; i < outputCount; ++i) { mLastChunks[i].SetNull(0); } // We need to generate at least one input uint16_t maxInputs = std::max(uint16_t(1), mEngine->InputCount()); OutputChunks inputChunks; inputChunks.SetLength(maxInputs); for (uint16_t i = 0; i < maxInputs; ++i) { ObtainInputBlock(inputChunks[i], i); } bool finished = false; if (maxInputs <= 1 && mEngine->OutputCount() <= 1) { mEngine->ProduceAudioBlock(this, inputChunks[0], &mLastChunks[0], &finished); } else { mEngine->ProduceAudioBlocksOnPorts(this, inputChunks, mLastChunks, &finished); } if (finished) { mMarkAsFinishedAfterThisBlock = true; } } if (mDisabledTrackIDs.Contains(AUDIO_NODE_STREAM_TRACK_ID)) { for (uint32_t i = 0; i < mLastChunks.Length(); ++i) { mLastChunks[i].SetNull(WEBAUDIO_BLOCK_SIZE); } } AdvanceOutputSegment(); }
void AudioNodeExternalInputStream::ProcessInput(GraphTime aFrom, GraphTime aTo, uint32_t aFlags) { // According to spec, number of outputs is always 1. MOZ_ASSERT(mLastChunks.Length() == 1); // GC stuff can result in our input stream being destroyed before this stream. // Handle that. if (!IsEnabled() || mInputs.IsEmpty() || mPassThrough) { mLastChunks[0].SetNull(WEBAUDIO_BLOCK_SIZE); AdvanceOutputSegment(); return; } MOZ_ASSERT(mInputs.Length() == 1); MediaStream* source = mInputs[0]->GetSource(); nsAutoTArray<AudioSegment,1> audioSegments; uint32_t inputChannels = 0; for (StreamBuffer::TrackIter tracks(source->mBuffer, MediaSegment::AUDIO); !tracks.IsEnded(); tracks.Next()) { const StreamBuffer::Track& inputTrack = *tracks; const AudioSegment& inputSegment = *static_cast<AudioSegment*>(inputTrack.GetSegment()); if (inputSegment.IsNull()) { continue; } AudioSegment& segment = *audioSegments.AppendElement(); GraphTime next; for (GraphTime t = aFrom; t < aTo; t = next) { MediaInputPort::InputInterval interval = mInputs[0]->GetNextInputInterval(t); interval.mEnd = std::min(interval.mEnd, aTo); if (interval.mStart >= interval.mEnd) break; next = interval.mEnd; StreamTime outputStart = GraphTimeToStreamTime(interval.mStart); StreamTime outputEnd = GraphTimeToStreamTime(interval.mEnd); StreamTime ticks = outputEnd - outputStart; if (interval.mInputIsBlocked) { segment.AppendNullData(ticks); } else { StreamTime inputStart = std::min(inputSegment.GetDuration(), source->GraphTimeToStreamTime(interval.mStart)); StreamTime inputEnd = std::min(inputSegment.GetDuration(), source->GraphTimeToStreamTime(interval.mEnd)); segment.AppendSlice(inputSegment, inputStart, inputEnd); // Pad if we're looking past the end of the track segment.AppendNullData(ticks - (inputEnd - inputStart)); } } for (AudioSegment::ChunkIterator iter(segment); !iter.IsEnded(); iter.Next()) { inputChannels = GetAudioChannelsSuperset(inputChannels, iter->ChannelCount()); } } uint32_t accumulateIndex = 0; if (inputChannels) { nsAutoTArray<float,GUESS_AUDIO_CHANNELS*WEBAUDIO_BLOCK_SIZE> downmixBuffer; for (uint32_t i = 0; i < audioSegments.Length(); ++i) { AudioChunk tmpChunk; ConvertSegmentToAudioBlock(&audioSegments[i], &tmpChunk, inputChannels); if (!tmpChunk.IsNull()) { if (accumulateIndex == 0) { AllocateAudioBlock(inputChannels, &mLastChunks[0]); } AccumulateInputChunk(accumulateIndex, tmpChunk, &mLastChunks[0], &downmixBuffer); accumulateIndex++; } } } if (accumulateIndex == 0) { mLastChunks[0].SetNull(WEBAUDIO_BLOCK_SIZE); } // Using AudioNodeStream's AdvanceOutputSegment to push the media stream graph along with null data. AdvanceOutputSegment(); }
// The MediaStreamGraph guarantees that this is actually one block, for // AudioNodeStreams. void AudioNodeStream::ProcessInput(GraphTime aFrom, GraphTime aTo, uint32_t aFlags) { uint16_t outputCount = mLastChunks.Length(); MOZ_ASSERT(outputCount == std::max(uint16_t(1), mEngine->OutputCount())); if (!mIsActive) { // mLastChunks are already null. #ifdef DEBUG for (const auto& chunk : mLastChunks) { MOZ_ASSERT(chunk.IsNull()); } #endif } else if (InMutedCycle()) { mInputChunks.Clear(); for (uint16_t i = 0; i < outputCount; ++i) { mLastChunks[i].SetNull(WEBAUDIO_BLOCK_SIZE); } } else { // We need to generate at least one input uint16_t maxInputs = std::max(uint16_t(1), mEngine->InputCount()); mInputChunks.SetLength(maxInputs); for (uint16_t i = 0; i < maxInputs; ++i) { ObtainInputBlock(mInputChunks[i], i); } bool finished = false; if (mPassThrough) { MOZ_ASSERT(outputCount == 1, "For now, we only support nodes that have one output port"); mLastChunks[0] = mInputChunks[0]; } else { if (maxInputs <= 1 && outputCount <= 1) { mEngine->ProcessBlock(this, aFrom, mInputChunks[0], &mLastChunks[0], &finished); } else { mEngine->ProcessBlocksOnPorts(this, mInputChunks, mLastChunks, &finished); } } for (uint16_t i = 0; i < outputCount; ++i) { NS_ASSERTION(mLastChunks[i].GetDuration() == WEBAUDIO_BLOCK_SIZE, "Invalid WebAudio chunk size"); } if (finished) { mMarkAsFinishedAfterThisBlock = true; if (mIsActive) { ScheduleCheckForInactive(); } } if (mDisabledTrackIDs.Contains(static_cast<TrackID>(AUDIO_TRACK))) { for (uint32_t i = 0; i < outputCount; ++i) { mLastChunks[i].SetNull(WEBAUDIO_BLOCK_SIZE); } } } if (!mFinished) { // Don't output anything while finished if (mFlags & EXTERNAL_OUTPUT) { AdvanceOutputSegment(); } if (mMarkAsFinishedAfterThisBlock && (aFlags & ALLOW_FINISH)) { // This stream was finished the last time that we looked at it, and all // of the depending streams have finished their output as well, so now // it's time to mark this stream as finished. if (mFlags & EXTERNAL_OUTPUT) { FinishOutput(); } FinishOnGraphThread(); } } }
// The MediaStreamGraph guarantees that this is actually one block, for // AudioNodeStreams. void AudioNodeStream::ProcessInput(GraphTime aFrom, GraphTime aTo, uint32_t aFlags) { if (!mFinished) { EnsureTrack(AUDIO_TRACK); } // No more tracks will be coming mBuffer.AdvanceKnownTracksTime(STREAM_TIME_MAX); uint16_t outputCount = mLastChunks.Length(); MOZ_ASSERT(outputCount == std::max(uint16_t(1), mEngine->OutputCount())); // Consider this stream blocked if it has already finished output. Normally // mBlocked would reflect this, but due to rounding errors our audio track may // appear to extend slightly beyond aFrom, so we might not be blocked yet. bool blocked = mFinished || mBlocked.GetAt(aFrom); // If the stream has finished at this time, it will be blocked. if (blocked || InMutedCycle()) { mInputChunks.Clear(); for (uint16_t i = 0; i < outputCount; ++i) { mLastChunks[i].SetNull(WEBAUDIO_BLOCK_SIZE); } } else { // We need to generate at least one input uint16_t maxInputs = std::max(uint16_t(1), mEngine->InputCount()); mInputChunks.SetLength(maxInputs); for (uint16_t i = 0; i < maxInputs; ++i) { ObtainInputBlock(mInputChunks[i], i); } bool finished = false; if (mPassThrough) { MOZ_ASSERT(outputCount == 1, "For now, we only support nodes that have one output port"); mLastChunks[0] = mInputChunks[0]; } else { if (maxInputs <= 1 && outputCount <= 1) { mEngine->ProcessBlock(this, mInputChunks[0], &mLastChunks[0], &finished); } else { mEngine->ProcessBlocksOnPorts(this, mInputChunks, mLastChunks, &finished); } } for (auto& chunk : mInputChunks) { // If the buffer is shared then it won't be reused, so release the // reference now. Keep the channel data array to save a free/alloc // pair. chunk.ReleaseBufferIfShared(); } for (uint16_t i = 0; i < outputCount; ++i) { NS_ASSERTION(mLastChunks[i].GetDuration() == WEBAUDIO_BLOCK_SIZE, "Invalid WebAudio chunk size"); } if (finished) { mMarkAsFinishedAfterThisBlock = true; } if (mDisabledTrackIDs.Contains(static_cast<TrackID>(AUDIO_TRACK))) { for (uint32_t i = 0; i < outputCount; ++i) { mLastChunks[i].SetNull(WEBAUDIO_BLOCK_SIZE); } } } if (!blocked) { // Don't output anything while blocked AdvanceOutputSegment(); if (mMarkAsFinishedAfterThisBlock && (aFlags & ALLOW_FINISH)) { // This stream was finished the last time that we looked at it, and all // of the depending streams have finished their output as well, so now // it's time to mark this stream as finished. FinishOutput(); } } }
void AudioNodeExternalInputStream::ProcessInput(GraphTime aFrom, GraphTime aTo, uint32_t aFlags) { // According to spec, number of outputs is always 1. mLastChunks.SetLength(1); // GC stuff can result in our input stream being destroyed before this stream. // Handle that. if (mInputs.IsEmpty()) { mLastChunks[0].SetNull(WEBAUDIO_BLOCK_SIZE); AdvanceOutputSegment(); return; } MOZ_ASSERT(mInputs.Length() == 1); MediaStream* source = mInputs[0]->GetSource(); nsAutoTArray<AudioSegment,1> audioSegments; nsAutoTArray<bool,1> trackMapEntriesUsed; uint32_t inputChannels = 0; for (StreamBuffer::TrackIter tracks(source->mBuffer, MediaSegment::AUDIO); !tracks.IsEnded(); tracks.Next()) { const StreamBuffer::Track& inputTrack = *tracks; // Create a TrackMapEntry if necessary. size_t trackMapIndex = GetTrackMapEntry(inputTrack, aFrom); // Maybe there's nothing in this track yet. If so, ignore it. (While the // track is only playing silence, we may not be able to determine the // correct number of channels to start resampling.) if (trackMapIndex == nsTArray<TrackMapEntry>::NoIndex) { continue; } while (trackMapEntriesUsed.Length() <= trackMapIndex) { trackMapEntriesUsed.AppendElement(false); } trackMapEntriesUsed[trackMapIndex] = true; TrackMapEntry* trackMap = &mTrackMap[trackMapIndex]; AudioSegment segment; GraphTime next; TrackRate inputTrackRate = inputTrack.GetRate(); for (GraphTime t = aFrom; t < aTo; t = next) { MediaInputPort::InputInterval interval = mInputs[0]->GetNextInputInterval(t); interval.mEnd = std::min(interval.mEnd, aTo); if (interval.mStart >= interval.mEnd) break; next = interval.mEnd; // Ticks >= startTicks and < endTicks are in the interval StreamTime outputEnd = GraphTimeToStreamTime(interval.mEnd); TrackTicks startTicks = trackMap->mSamplesPassedToResampler + segment.GetDuration(); StreamTime outputStart = GraphTimeToStreamTime(interval.mStart); NS_ASSERTION(startTicks == TimeToTicksRoundUp(inputTrackRate, outputStart), "Samples missing"); TrackTicks endTicks = TimeToTicksRoundUp(inputTrackRate, outputEnd); TrackTicks ticks = endTicks - startTicks; if (interval.mInputIsBlocked) { segment.AppendNullData(ticks); } else { // See comments in TrackUnionStream::CopyTrackData StreamTime inputStart = source->GraphTimeToStreamTime(interval.mStart); StreamTime inputEnd = source->GraphTimeToStreamTime(interval.mEnd); TrackTicks inputTrackEndPoint = inputTrack.IsEnded() ? inputTrack.GetEnd() : TRACK_TICKS_MAX; if (trackMap->mEndOfLastInputIntervalInInputStream != inputStart || trackMap->mEndOfLastInputIntervalInOutputStream != outputStart) { // Start of a new series of intervals where neither stream is blocked. trackMap->mEndOfConsumedInputTicks = TimeToTicksRoundDown(inputTrackRate, inputStart) - 1; } TrackTicks inputStartTicks = trackMap->mEndOfConsumedInputTicks; TrackTicks inputEndTicks = inputStartTicks + ticks; trackMap->mEndOfConsumedInputTicks = inputEndTicks; trackMap->mEndOfLastInputIntervalInInputStream = inputEnd; trackMap->mEndOfLastInputIntervalInOutputStream = outputEnd; if (inputStartTicks < 0) { // Data before the start of the track is just null. segment.AppendNullData(-inputStartTicks); inputStartTicks = 0; } if (inputEndTicks > inputStartTicks) { segment.AppendSlice(*inputTrack.GetSegment(), std::min(inputTrackEndPoint, inputStartTicks), std::min(inputTrackEndPoint, inputEndTicks)); } // Pad if we're looking past the end of the track segment.AppendNullData(ticks - segment.GetDuration()); } } trackMap->mSamplesPassedToResampler += segment.GetDuration(); trackMap->ResampleInputData(&segment); if (trackMap->mResampledData.GetDuration() < mCurrentOutputPosition + WEBAUDIO_BLOCK_SIZE) { // We don't have enough data. Delay it. trackMap->mResampledData.InsertNullDataAtStart( mCurrentOutputPosition + WEBAUDIO_BLOCK_SIZE - trackMap->mResampledData.GetDuration()); } audioSegments.AppendElement()->AppendSlice(trackMap->mResampledData, mCurrentOutputPosition, mCurrentOutputPosition + WEBAUDIO_BLOCK_SIZE); trackMap->mResampledData.ForgetUpTo(mCurrentOutputPosition + WEBAUDIO_BLOCK_SIZE); inputChannels = GetAudioChannelsSuperset(inputChannels, trackMap->mResamplerChannelCount); } for (int32_t i = mTrackMap.Length() - 1; i >= 0; --i) { if (i >= int32_t(trackMapEntriesUsed.Length()) || !trackMapEntriesUsed[i]) { mTrackMap.RemoveElementAt(i); } } uint32_t accumulateIndex = 0; if (inputChannels) { nsAutoTArray<float,GUESS_AUDIO_CHANNELS*WEBAUDIO_BLOCK_SIZE> downmixBuffer; for (uint32_t i = 0; i < audioSegments.Length(); ++i) { AudioChunk tmpChunk; ConvertSegmentToAudioBlock(&audioSegments[i], &tmpChunk); if (!tmpChunk.IsNull()) { if (accumulateIndex == 0) { AllocateAudioBlock(inputChannels, &mLastChunks[0]); } AccumulateInputChunk(accumulateIndex, tmpChunk, &mLastChunks[0], &downmixBuffer); accumulateIndex++; } } } if (accumulateIndex == 0) { mLastChunks[0].SetNull(WEBAUDIO_BLOCK_SIZE); } mCurrentOutputPosition += WEBAUDIO_BLOCK_SIZE; // Using AudioNodeStream's AdvanceOutputSegment to push the media stream graph along with null data. AdvanceOutputSegment(); }