void AudioCaptureStream::MixerCallback(AudioDataValue* aMixedBuffer, AudioSampleFormat aFormat, uint32_t aChannels, uint32_t aFrames, uint32_t aSampleRate) { AutoTArray<nsTArray<AudioDataValue>, MONO> output; AutoTArray<const AudioDataValue*, MONO> bufferPtrs; output.SetLength(MONO); bufferPtrs.SetLength(MONO); uint32_t written = 0; // We need to copy here, because the mixer will reuse the storage, we should // not hold onto it. Buffers are in planar format. for (uint32_t channel = 0; channel < aChannels; channel++) { AudioDataValue* out = output[channel].AppendElements(aFrames); PodCopy(out, aMixedBuffer + written, aFrames); bufferPtrs[channel] = out; written += aFrames; } AudioChunk chunk; chunk.mBuffer = new mozilla::SharedChannelArrayBuffer<AudioDataValue>(&output); chunk.mDuration = aFrames; chunk.mBufferFormat = aFormat; chunk.mVolume = 1.0f; chunk.mChannelData.SetLength(MONO); for (uint32_t channel = 0; channel < aChannels; channel++) { chunk.mChannelData[channel] = bufferPtrs[channel]; } // Now we have mixed data, simply append it to out track. EnsureTrack(mTrackId)->Get<AudioSegment>()->AppendAndConsumeChunk(&chunk); }
// The MediaStreamGraph guarantees that this is actually one block, for // AudioNodeStreams. void AudioNodeStream::ProcessInput(GraphTime aFrom, GraphTime aTo, uint32_t aFlags) { EnsureTrack(AUDIO_TRACK, mSampleRate); // No more tracks will be coming mBuffer.AdvanceKnownTracksTime(STREAM_TIME_MAX); uint16_t outputCount = std::max(uint16_t(1), mEngine->OutputCount()); mLastChunks.SetLength(outputCount); // Consider this stream blocked if it has already finished output. Normally // mBlocked would reflect this, but due to rounding errors our audio track may // appear to extend slightly beyond aFrom, so we might not be blocked yet. bool blocked = mFinished || mBlocked.GetAt(aFrom); // If the stream has finished at this time, it will be blocked. if (mMuted || blocked) { for (uint16_t i = 0; i < outputCount; ++i) { mLastChunks[i].SetNull(WEBAUDIO_BLOCK_SIZE); } } else { // We need to generate at least one input uint16_t maxInputs = std::max(uint16_t(1), mEngine->InputCount()); OutputChunks inputChunks; inputChunks.SetLength(maxInputs); for (uint16_t i = 0; i < maxInputs; ++i) { ObtainInputBlock(inputChunks[i], i); } bool finished = false; if (maxInputs <= 1 && mEngine->OutputCount() <= 1) { mEngine->ProcessBlock(this, inputChunks[0], &mLastChunks[0], &finished); } else { mEngine->ProcessBlocksOnPorts(this, inputChunks, mLastChunks, &finished); } for (uint16_t i = 0; i < outputCount; ++i) { NS_ASSERTION(mLastChunks[i].GetDuration() == WEBAUDIO_BLOCK_SIZE, "Invalid WebAudio chunk size"); } if (finished) { mMarkAsFinishedAfterThisBlock = true; } if (mDisabledTrackIDs.Contains(static_cast<TrackID>(AUDIO_TRACK))) { for (uint32_t i = 0; i < outputCount; ++i) { mLastChunks[i].SetNull(WEBAUDIO_BLOCK_SIZE); } } } if (!blocked) { // Don't output anything while blocked AdvanceOutputSegment(); if (mMarkAsFinishedAfterThisBlock && (aFlags & ALLOW_FINISH)) { // This stream was finished the last time that we looked at it, and all // of the depending streams have finished their output as well, so now // it's time to mark this stream as finished. FinishOutput(); } } }
virtual void Run() override { auto ns = static_cast<AudioNodeStream*>(mStream); ns->mBufferStartTime -= mAdvance; StreamBuffer::Track* track = ns->EnsureTrack(AUDIO_TRACK); track->Get<AudioSegment>()->AppendNullData(mAdvance); ns->GraphImpl()->DecrementSuspendCount(mStream); }
// The MediaStreamGraph guarantees that this is actually one block, for // AudioNodeStreams. void AudioNodeStream::ProduceOutput(GraphTime aFrom, GraphTime aTo) { if (mMarkAsFinishedAfterThisBlock) { // This stream was finished the last time that we looked at it, and all // of the depending streams have finished their output as well, so now // it's time to mark this stream as finished. FinishOutput(); } StreamBuffer::Track* track = EnsureTrack(); AudioSegment* segment = track->Get<AudioSegment>(); mLastChunks.SetLength(1); mLastChunks[0].SetNull(0); if (mInCycle) { // XXX DelayNode not supported yet so just produce silence mLastChunks[0].SetNull(WEBAUDIO_BLOCK_SIZE); } else { // We need to generate at least one input uint16_t maxInputs = std::max(uint16_t(1), mEngine->InputCount()); OutputChunks inputChunks; inputChunks.SetLength(maxInputs); for (uint16_t i = 0; i < maxInputs; ++i) { ObtainInputBlock(inputChunks[i], i); } bool finished = false; if (maxInputs <= 1 && mEngine->OutputCount() <= 1) { mEngine->ProduceAudioBlock(this, inputChunks[0], &mLastChunks[0], &finished); } else { mEngine->ProduceAudioBlocksOnPorts(this, inputChunks, mLastChunks, &finished); } if (finished) { mMarkAsFinishedAfterThisBlock = true; } } if (mKind == MediaStreamGraph::EXTERNAL_STREAM) { segment->AppendAndConsumeChunk(&mLastChunks[0]); } else { segment->AppendNullData(mLastChunks[0].GetDuration()); } for (uint32_t j = 0; j < mListeners.Length(); ++j) { MediaStreamListener* l = mListeners[j]; AudioChunk copyChunk = mLastChunks[0]; AudioSegment tmpSegment; tmpSegment.AppendAndConsumeChunk(©Chunk); l->NotifyQueuedTrackChanges(Graph(), AUDIO_NODE_STREAM_TRACK_ID, IdealAudioRate(), segment->GetDuration(), 0, tmpSegment); } }
void AudioCaptureStream::ProcessInput(GraphTime aFrom, GraphTime aTo, uint32_t aFlags) { uint32_t inputCount = mInputs.Length(); StreamBuffer::Track* track = EnsureTrack(mTrackId); // Notify the DOM everything is in order. if (!mTrackCreated) { for (uint32_t i = 0; i < mListeners.Length(); i++) { MediaStreamListener* l = mListeners[i]; AudioSegment tmp; l->NotifyQueuedTrackChanges( Graph(), mTrackId, 0, MediaStreamListener::TRACK_EVENT_CREATED, tmp); l->NotifyFinishedTrackCreation(Graph()); } mTrackCreated = true; } // If the captured stream is connected back to a object on the page (be it an // HTMLMediaElement with a stream as source, or an AudioContext), a cycle // situation occur. This can work if it's an AudioContext with at least one // DelayNode, but the MSG will mute the whole cycle otherwise. if (mFinished || InMutedCycle() || inputCount == 0) { track->Get<AudioSegment>()->AppendNullData(aTo - aFrom); } else { // We mix down all the tracks of all inputs, to a stereo track. Everything // is {up,down}-mixed to stereo. mMixer.StartMixing(); AudioSegment output; for (uint32_t i = 0; i < inputCount; i++) { MediaStream* s = mInputs[i]->GetSource(); StreamBuffer::TrackIter tracks(s->GetStreamBuffer(), MediaSegment::AUDIO); while (!tracks.IsEnded()) { AudioSegment* inputSegment = tracks->Get<AudioSegment>(); StreamTime inputStart = s->GraphTimeToStreamTimeWithBlocking(aFrom); StreamTime inputEnd = s->GraphTimeToStreamTimeWithBlocking(aTo); AudioSegment toMix; toMix.AppendSlice(*inputSegment, inputStart, inputEnd); // Care for streams blocked in the [aTo, aFrom] range. if (inputEnd - inputStart < aTo - aFrom) { toMix.AppendNullData((aTo - aFrom) - (inputEnd - inputStart)); } toMix.Mix(mMixer, MONO, Graph()->GraphRate()); tracks.Next(); } } // This calls MixerCallback below mMixer.FinishMixing(); } // Regardless of the status of the input tracks, we go foward. mBuffer.AdvanceKnownTracksTime(GraphTimeToStreamTimeWithBlocking((aTo))); }
// The MediaStreamGraph guarantees that this is actually one block, for // AudioNodeStreams. void AudioNodeStream::ProduceOutput(GraphTime aFrom, GraphTime aTo) { if (mMarkAsFinishedAfterThisBlock) { // This stream was finished the last time that we looked at it, and all // of the depending streams have finished their output as well, so now // it's time to mark this stream as finished. FinishOutput(); } EnsureTrack(AUDIO_NODE_STREAM_TRACK_ID, mSampleRate); uint16_t outputCount = std::max(uint16_t(1), mEngine->OutputCount()); mLastChunks.SetLength(outputCount); if (mInCycle) { // XXX DelayNode not supported yet so just produce silence for (uint16_t i = 0; i < outputCount; ++i) { mLastChunks[i].SetNull(WEBAUDIO_BLOCK_SIZE); } } else { for (uint16_t i = 0; i < outputCount; ++i) { mLastChunks[i].SetNull(0); } // We need to generate at least one input uint16_t maxInputs = std::max(uint16_t(1), mEngine->InputCount()); OutputChunks inputChunks; inputChunks.SetLength(maxInputs); for (uint16_t i = 0; i < maxInputs; ++i) { ObtainInputBlock(inputChunks[i], i); } bool finished = false; if (maxInputs <= 1 && mEngine->OutputCount() <= 1) { mEngine->ProduceAudioBlock(this, inputChunks[0], &mLastChunks[0], &finished); } else { mEngine->ProduceAudioBlocksOnPorts(this, inputChunks, mLastChunks, &finished); } if (finished) { mMarkAsFinishedAfterThisBlock = true; } } if (mDisabledTrackIDs.Contains(AUDIO_NODE_STREAM_TRACK_ID)) { for (uint32_t i = 0; i < mLastChunks.Length(); ++i) { mLastChunks[i].SetNull(WEBAUDIO_BLOCK_SIZE); } } AdvanceOutputSegment(); }
void AudioNodeStream::FinishOutput() { StreamBuffer::Track* track = EnsureTrack(AUDIO_TRACK); track->SetEnded(); for (uint32_t j = 0; j < mListeners.Length(); ++j) { MediaStreamListener* l = mListeners[j]; AudioSegment emptySegment; l->NotifyQueuedTrackChanges(Graph(), AUDIO_TRACK, track->GetSegment()->GetDuration(), MediaStreamListener::TRACK_EVENT_ENDED, emptySegment); } }
void AudioNodeStream::FinishOutput() { if (IsFinishedOnGraphThread()) { return; } StreamBuffer::Track* track = EnsureTrack(); track->SetEnded(); FinishOnGraphThread(); for (uint32_t j = 0; j < mListeners.Length(); ++j) { MediaStreamListener* l = mListeners[j]; AudioSegment emptySegment; l->NotifyQueuedTrackChanges(Graph(), AUDIO_NODE_STREAM_TRACK_ID, IdealAudioRate(), track->GetSegment()->GetDuration(), MediaStreamListener::TRACK_EVENT_ENDED, emptySegment); } }
void AudioNodeStream::AdvanceOutputSegment() { StreamBuffer::Track* track = EnsureTrack(AUDIO_TRACK); AudioSegment* segment = track->Get<AudioSegment>(); if (mKind == MediaStreamGraph::EXTERNAL_STREAM) { segment->AppendAndConsumeChunk(&mLastChunks[0]); } else { segment->AppendNullData(mLastChunks[0].GetDuration()); } for (uint32_t j = 0; j < mListeners.Length(); ++j) { MediaStreamListener* l = mListeners[j]; AudioChunk copyChunk = mLastChunks[0]; AudioSegment tmpSegment; tmpSegment.AppendAndConsumeChunk(©Chunk); l->NotifyQueuedTrackChanges(Graph(), AUDIO_TRACK, segment->GetDuration(), 0, tmpSegment); } }
// The MediaStreamGraph guarantees that this is actually one block, for // AudioNodeStreams. void AudioNodeStream::ProduceOutput(GraphTime aFrom, GraphTime aTo) { StreamBuffer::Track* track = EnsureTrack(); AudioChunk outputChunk; AudioSegment* segment = track->Get<AudioSegment>(); outputChunk.SetNull(0); if (mInCycle) { // XXX DelayNode not supported yet so just produce silence outputChunk.SetNull(WEBAUDIO_BLOCK_SIZE); } else { AudioChunk tmpChunk; AudioChunk* inputChunk = ObtainInputBlock(&tmpChunk); bool finished = false; mEngine->ProduceAudioBlock(this, *inputChunk, &outputChunk, &finished); if (finished) { FinishOutput(); } } mLastChunk = outputChunk; if (mKind == MediaStreamGraph::EXTERNAL_STREAM) { segment->AppendAndConsumeChunk(&outputChunk); } else { segment->AppendNullData(outputChunk.GetDuration()); } for (uint32_t j = 0; j < mListeners.Length(); ++j) { MediaStreamListener* l = mListeners[j]; AudioChunk copyChunk = outputChunk; AudioSegment tmpSegment; tmpSegment.AppendAndConsumeChunk(©Chunk); l->NotifyQueuedTrackChanges(Graph(), AUDIO_NODE_STREAM_TRACK_ID, IdealAudioRate(), segment->GetDuration(), 0, tmpSegment); } }
void AudioNodeStream::AdvanceOutputSegment() { StreamBuffer::Track* track = EnsureTrack(AUDIO_TRACK); // No more tracks will be coming mBuffer.AdvanceKnownTracksTime(STREAM_TIME_MAX); AudioSegment* segment = track->Get<AudioSegment>(); if (!mLastChunks[0].IsNull()) { segment->AppendAndConsumeChunk(mLastChunks[0].AsMutableChunk()); } else { segment->AppendNullData(mLastChunks[0].GetDuration()); } for (uint32_t j = 0; j < mListeners.Length(); ++j) { MediaStreamListener* l = mListeners[j]; AudioChunk copyChunk = mLastChunks[0].AsAudioChunk(); AudioSegment tmpSegment; tmpSegment.AppendAndConsumeChunk(©Chunk); l->NotifyQueuedTrackChanges(Graph(), AUDIO_TRACK, segment->GetDuration(), 0, tmpSegment); } }
TrackTicks AudioNodeStream::GetCurrentPosition() { return EnsureTrack(AUDIO_TRACK, mSampleRate)->Get<AudioSegment>()->GetDuration(); }
TrackTicks AudioNodeStream::GetCurrentPosition() { return EnsureTrack()->Get<AudioSegment>()->GetDuration(); }
StreamTime AudioNodeStream::GetCurrentPosition() { NS_ASSERTION(!mFinished, "Don't create another track after finishing"); return EnsureTrack(AUDIO_TRACK)->Get<AudioSegment>()->GetDuration(); }
// The MediaStreamGraph guarantees that this is actually one block, for // AudioNodeStreams. void AudioNodeStream::ProcessInput(GraphTime aFrom, GraphTime aTo, uint32_t aFlags) { if (!mFinished) { EnsureTrack(AUDIO_TRACK); } // No more tracks will be coming mBuffer.AdvanceKnownTracksTime(STREAM_TIME_MAX); uint16_t outputCount = mLastChunks.Length(); MOZ_ASSERT(outputCount == std::max(uint16_t(1), mEngine->OutputCount())); // Consider this stream blocked if it has already finished output. Normally // mBlocked would reflect this, but due to rounding errors our audio track may // appear to extend slightly beyond aFrom, so we might not be blocked yet. bool blocked = mFinished || mBlocked.GetAt(aFrom); // If the stream has finished at this time, it will be blocked. if (blocked || InMutedCycle()) { mInputChunks.Clear(); for (uint16_t i = 0; i < outputCount; ++i) { mLastChunks[i].SetNull(WEBAUDIO_BLOCK_SIZE); } } else { // We need to generate at least one input uint16_t maxInputs = std::max(uint16_t(1), mEngine->InputCount()); mInputChunks.SetLength(maxInputs); for (uint16_t i = 0; i < maxInputs; ++i) { ObtainInputBlock(mInputChunks[i], i); } bool finished = false; if (mPassThrough) { MOZ_ASSERT(outputCount == 1, "For now, we only support nodes that have one output port"); mLastChunks[0] = mInputChunks[0]; } else { if (maxInputs <= 1 && outputCount <= 1) { mEngine->ProcessBlock(this, mInputChunks[0], &mLastChunks[0], &finished); } else { mEngine->ProcessBlocksOnPorts(this, mInputChunks, mLastChunks, &finished); } } for (auto& chunk : mInputChunks) { // If the buffer is shared then it won't be reused, so release the // reference now. Keep the channel data array to save a free/alloc // pair. chunk.ReleaseBufferIfShared(); } for (uint16_t i = 0; i < outputCount; ++i) { NS_ASSERTION(mLastChunks[i].GetDuration() == WEBAUDIO_BLOCK_SIZE, "Invalid WebAudio chunk size"); } if (finished) { mMarkAsFinishedAfterThisBlock = true; } if (mDisabledTrackIDs.Contains(static_cast<TrackID>(AUDIO_TRACK))) { for (uint32_t i = 0; i < outputCount; ++i) { mLastChunks[i].SetNull(WEBAUDIO_BLOCK_SIZE); } } } if (!blocked) { // Don't output anything while blocked AdvanceOutputSegment(); if (mMarkAsFinishedAfterThisBlock && (aFlags & ALLOW_FINISH)) { // This stream was finished the last time that we looked at it, and all // of the depending streams have finished their output as well, so now // it's time to mark this stream as finished. FinishOutput(); } } }