nsresult OmxAudioTrackEncoder::GetEncodedTrack(EncodedFrameContainer& aData) { PROFILER_LABEL("OmxAACAudioTrackEncoder", "GetEncodedTrack", js::ProfileEntry::Category::OTHER); AudioSegment segment; // Move all the samples from mRawSegment to segment. We only hold // the monitor in this block. { ReentrantMonitorAutoEnter mon(mReentrantMonitor); // Wait if mEncoder is not initialized nor canceled. while (!mInitialized && !mCanceled) { mReentrantMonitor.Wait(); } if (mCanceled || mEncodingComplete) { return NS_ERROR_FAILURE; } segment.AppendFrom(&mRawSegment); } nsresult rv; if (segment.GetDuration() == 0) { // Notify EOS at least once, even if segment is empty. if (mEndOfStream && !mEosSetInEncoder) { mEosSetInEncoder = true; rv = mEncoder->Encode(segment, OMXCodecWrapper::BUFFER_EOS); NS_ENSURE_SUCCESS(rv, rv); } // Nothing to encode but encoder could still have encoded data for earlier // input. return AppendEncodedFrames(aData); } // OMX encoder has limited input buffers only so we have to feed input and get // output more than once if there are too many samples pending in segment. while (segment.GetDuration() > 0) { rv = mEncoder->Encode(segment, mEndOfStream ? OMXCodecWrapper::BUFFER_EOS : 0); NS_ENSURE_SUCCESS(rv, rv); rv = AppendEncodedFrames(aData); NS_ENSURE_SUCCESS(rv, rv); } return NS_OK; }
// The MediaStreamGraph guarantees that this is actually one block, for // AudioNodeStreams. void AudioNodeStream::ProduceOutput(GraphTime aFrom, GraphTime aTo) { if (mMarkAsFinishedAfterThisBlock) { // This stream was finished the last time that we looked at it, and all // of the depending streams have finished their output as well, so now // it's time to mark this stream as finished. FinishOutput(); } StreamBuffer::Track* track = EnsureTrack(); AudioSegment* segment = track->Get<AudioSegment>(); mLastChunks.SetLength(1); mLastChunks[0].SetNull(0); if (mInCycle) { // XXX DelayNode not supported yet so just produce silence mLastChunks[0].SetNull(WEBAUDIO_BLOCK_SIZE); } else { // We need to generate at least one input uint16_t maxInputs = std::max(uint16_t(1), mEngine->InputCount()); OutputChunks inputChunks; inputChunks.SetLength(maxInputs); for (uint16_t i = 0; i < maxInputs; ++i) { ObtainInputBlock(inputChunks[i], i); } bool finished = false; if (maxInputs <= 1 && mEngine->OutputCount() <= 1) { mEngine->ProduceAudioBlock(this, inputChunks[0], &mLastChunks[0], &finished); } else { mEngine->ProduceAudioBlocksOnPorts(this, inputChunks, mLastChunks, &finished); } if (finished) { mMarkAsFinishedAfterThisBlock = true; } } if (mKind == MediaStreamGraph::EXTERNAL_STREAM) { segment->AppendAndConsumeChunk(&mLastChunks[0]); } else { segment->AppendNullData(mLastChunks[0].GetDuration()); } for (uint32_t j = 0; j < mListeners.Length(); ++j) { MediaStreamListener* l = mListeners[j]; AudioChunk copyChunk = mLastChunks[0]; AudioSegment tmpSegment; tmpSegment.AppendAndConsumeChunk(©Chunk); l->NotifyQueuedTrackChanges(Graph(), AUDIO_NODE_STREAM_TRACK_ID, IdealAudioRate(), segment->GetDuration(), 0, tmpSegment); } }
void DecodedStream::SendAudio(double aVolume, bool aIsSameOrigin, const PrincipalHandle& aPrincipalHandle) { AssertOwnerThread(); if (!mInfo.HasAudio()) { return; } AudioSegment output; uint32_t rate = mInfo.mAudio.mRate; AutoTArray<RefPtr<MediaData>,10> audio; TrackID audioTrackId = mInfo.mAudio.mTrackId; SourceMediaStream* sourceStream = mData->mStream; // It's OK to hold references to the AudioData because AudioData // is ref-counted. mAudioQueue.GetElementsAfter(mData->mNextAudioTime, &audio); for (uint32_t i = 0; i < audio.Length(); ++i) { SendStreamAudio(mData.get(), mStartTime.ref(), audio[i], &output, rate, aPrincipalHandle); } output.ApplyVolume(aVolume); if (!aIsSameOrigin) { output.ReplaceWithDisabled(); } // |mNextAudioTime| is updated as we process each audio sample in // SendStreamAudio(). This is consistent with how |mNextVideoTime| // is updated for video samples. if (output.GetDuration() > 0) { sourceStream->AppendToTrack(audioTrackId, &output); } if (mAudioQueue.IsFinished() && !mData->mHaveSentFinishAudio) { sourceStream->EndTrack(audioTrackId); mData->mHaveSentFinishAudio = true; } }
void AudioNodeStream::AdvanceOutputSegment() { StreamBuffer::Track* track = EnsureTrack(AUDIO_TRACK); AudioSegment* segment = track->Get<AudioSegment>(); if (mKind == MediaStreamGraph::EXTERNAL_STREAM) { segment->AppendAndConsumeChunk(&mLastChunks[0]); } else { segment->AppendNullData(mLastChunks[0].GetDuration()); } for (uint32_t j = 0; j < mListeners.Length(); ++j) { MediaStreamListener* l = mListeners[j]; AudioChunk copyChunk = mLastChunks[0]; AudioSegment tmpSegment; tmpSegment.AppendAndConsumeChunk(©Chunk); l->NotifyQueuedTrackChanges(Graph(), AUDIO_TRACK, segment->GetDuration(), 0, tmpSegment); } }
// The MediaStreamGraph guarantees that this is actually one block, for // AudioNodeStreams. void AudioNodeStream::ProduceOutput(GraphTime aFrom, GraphTime aTo) { StreamBuffer::Track* track = EnsureTrack(); AudioChunk outputChunk; AudioSegment* segment = track->Get<AudioSegment>(); outputChunk.SetNull(0); if (mInCycle) { // XXX DelayNode not supported yet so just produce silence outputChunk.SetNull(WEBAUDIO_BLOCK_SIZE); } else { AudioChunk tmpChunk; AudioChunk* inputChunk = ObtainInputBlock(&tmpChunk); bool finished = false; mEngine->ProduceAudioBlock(this, *inputChunk, &outputChunk, &finished); if (finished) { FinishOutput(); } } mLastChunk = outputChunk; if (mKind == MediaStreamGraph::EXTERNAL_STREAM) { segment->AppendAndConsumeChunk(&outputChunk); } else { segment->AppendNullData(outputChunk.GetDuration()); } for (uint32_t j = 0; j < mListeners.Length(); ++j) { MediaStreamListener* l = mListeners[j]; AudioChunk copyChunk = outputChunk; AudioSegment tmpSegment; tmpSegment.AppendAndConsumeChunk(©Chunk); l->NotifyQueuedTrackChanges(Graph(), AUDIO_NODE_STREAM_TRACK_ID, IdealAudioRate(), segment->GetDuration(), 0, tmpSegment); } }
void AudioNodeStream::AdvanceOutputSegment() { StreamBuffer::Track* track = EnsureTrack(AUDIO_TRACK); // No more tracks will be coming mBuffer.AdvanceKnownTracksTime(STREAM_TIME_MAX); AudioSegment* segment = track->Get<AudioSegment>(); if (!mLastChunks[0].IsNull()) { segment->AppendAndConsumeChunk(mLastChunks[0].AsMutableChunk()); } else { segment->AppendNullData(mLastChunks[0].GetDuration()); } for (uint32_t j = 0; j < mListeners.Length(); ++j) { MediaStreamListener* l = mListeners[j]; AudioChunk copyChunk = mLastChunks[0].AsAudioChunk(); AudioSegment tmpSegment; tmpSegment.AppendAndConsumeChunk(©Chunk); l->NotifyQueuedTrackChanges(Graph(), AUDIO_TRACK, segment->GetDuration(), 0, tmpSegment); } }
TEST(OpusAudioTrackEncoder, Init) { { // The encoder does not normally recieve enough info from null data to // init. However, multiple attempts to do so, with sufficiently long // duration segments, should result in a best effort attempt. The first // attempt should never do this though, even if the duration is long: OpusTrackEncoder encoder(48000); AudioSegment segment; segment.AppendNullData(48000 * 100); encoder.TryInit(segment, segment.GetDuration()); EXPECT_FALSE(encoder.IsInitialized()); // Multiple init attempts should result in best effort init: encoder.TryInit(segment, segment.GetDuration()); EXPECT_TRUE(encoder.IsInitialized()); } { // If the duration of the segments given to the encoder is not long then // we shouldn't try a best effort init: OpusTrackEncoder encoder(48000); AudioSegment segment; segment.AppendNullData(1); encoder.TryInit(segment, segment.GetDuration()); EXPECT_FALSE(encoder.IsInitialized()); encoder.TryInit(segment, segment.GetDuration()); EXPECT_FALSE(encoder.IsInitialized()); } { // For non-null segments we should init immediately OpusTrackEncoder encoder(48000); AudioSegment segment; AudioGenerator generator(2, 48000); generator.Generate(segment, 1); encoder.TryInit(segment, segment.GetDuration()); EXPECT_TRUE(encoder.IsInitialized()); } { // Test low sample rate bound OpusTrackEncoder encoder(7999); AudioSegment segment; AudioGenerator generator(2, 7999); generator.Generate(segment, 1); encoder.TryInit(segment, segment.GetDuration()); EXPECT_FALSE(encoder.IsInitialized()); } { // Test low sample rate bound OpusTrackEncoder encoder(8000); AudioSegment segment; AudioGenerator generator(2, 8000); generator.Generate(segment, 1); encoder.TryInit(segment, segment.GetDuration()); EXPECT_TRUE(encoder.IsInitialized()); } { // Test high sample rate bound OpusTrackEncoder encoder(192001); AudioSegment segment; AudioGenerator generator(2, 192001); generator.Generate(segment, 1); encoder.TryInit(segment, segment.GetDuration()); EXPECT_FALSE(encoder.IsInitialized()); } { // Test high sample rate bound OpusTrackEncoder encoder(192000); AudioSegment segment; AudioGenerator generator(2, 192000); generator.Generate(segment, 1); encoder.TryInit(segment, segment.GetDuration()); EXPECT_TRUE(encoder.IsInitialized()); } }
void AudioCaptureStream::ProcessInput(GraphTime aFrom, GraphTime aTo, uint32_t aFlags) { if (!mStarted) { return; } uint32_t inputCount = mInputs.Length(); StreamTracks::Track* track = EnsureTrack(mTrackId); // Notify the DOM everything is in order. if (!mTrackCreated) { for (uint32_t i = 0; i < mListeners.Length(); i++) { MediaStreamListener* l = mListeners[i]; AudioSegment tmp; l->NotifyQueuedTrackChanges( Graph(), mTrackId, 0, TrackEventCommand::TRACK_EVENT_CREATED, tmp); l->NotifyFinishedTrackCreation(Graph()); } mTrackCreated = true; } if (IsFinishedOnGraphThread()) { return; } // If the captured stream is connected back to a object on the page (be it an // HTMLMediaElement with a stream as source, or an AudioContext), a cycle // situation occur. This can work if it's an AudioContext with at least one // DelayNode, but the MSG will mute the whole cycle otherwise. if (InMutedCycle() || inputCount == 0) { track->Get<AudioSegment>()->AppendNullData(aTo - aFrom); } else { // We mix down all the tracks of all inputs, to a stereo track. Everything // is {up,down}-mixed to stereo. mMixer.StartMixing(); AudioSegment output; for (uint32_t i = 0; i < inputCount; i++) { MediaStream* s = mInputs[i]->GetSource(); StreamTracks::TrackIter tracks(s->GetStreamTracks(), MediaSegment::AUDIO); while (!tracks.IsEnded()) { AudioSegment* inputSegment = tracks->Get<AudioSegment>(); StreamTime inputStart = s->GraphTimeToStreamTimeWithBlocking(aFrom); StreamTime inputEnd = s->GraphTimeToStreamTimeWithBlocking(aTo); if (tracks->IsEnded() && inputSegment->GetDuration() <= inputEnd) { // If the input track has ended and we have consumed all its data it // can be ignored. continue; } AudioSegment toMix; toMix.AppendSlice(*inputSegment, inputStart, inputEnd); // Care for streams blocked in the [aTo, aFrom] range. if (inputEnd - inputStart < aTo - aFrom) { toMix.AppendNullData((aTo - aFrom) - (inputEnd - inputStart)); } toMix.Mix(mMixer, MONO, Graph()->GraphRate()); tracks.Next(); } } // This calls MixerCallback below mMixer.FinishMixing(); } // Regardless of the status of the input tracks, we go foward. mTracks.AdvanceKnownTracksTime(GraphTimeToStreamTimeWithBlocking((aTo))); }
nsresult OMXAudioEncoder::Encode(AudioSegment& aSegment, int aInputFlags, bool* aSendEOS) { #ifndef MOZ_SAMPLE_TYPE_S16 #error MediaCodec accepts only 16-bit PCM data. #endif MOZ_ASSERT(mStarted, "Configure() should be called before Encode()."); size_t numSamples = aSegment.GetDuration(); // Get input buffer. InputBufferHelper buffer(mCodec, mInputBufs, *this, aInputFlags); status_t result = buffer.Dequeue(); if (result == -EAGAIN) { // All input buffers are full. Caller can try again later after consuming // some output buffers. return NS_OK; } NS_ENSURE_TRUE(result == OK, NS_ERROR_FAILURE); size_t sourceSamplesCopied = 0; // Number of copied samples. if (numSamples > 0) { // Copy input PCM data to input buffer until queue is empty. AudioSegment::ChunkIterator iter(const_cast<AudioSegment&>(aSegment)); while (!iter.IsEnded()) { BufferState result = buffer.ReadChunk(*iter, &sourceSamplesCopied); if (result == WAIT_FOR_NEW_BUFFER) { // All input buffers are full. Caller can try again later after // consuming some output buffers. aSegment.RemoveLeading(sourceSamplesCopied); return NS_OK; } else if (result == BUFFER_FAIL) { return NS_ERROR_FAILURE; } else { iter.Next(); } } // Remove the samples already been copied into buffer if (sourceSamplesCopied > 0) { aSegment.RemoveLeading(sourceSamplesCopied); } } else if (aInputFlags & BUFFER_EOS) { buffer.SendEOSToBuffer(&sourceSamplesCopied); } // Enqueue the remaining data to buffer MOZ_ASSERT(sourceSamplesCopied > 0, "No data needs to be enqueued!"); int flags = aInputFlags; if (aSegment.GetDuration() > 0) { // Don't signal EOS until source segment is empty. flags &= ~BUFFER_EOS; } result = buffer.Enqueue(mTimestamp, flags); NS_ENSURE_TRUE(result == OK, NS_ERROR_FAILURE); if (aSendEOS && (aInputFlags & BUFFER_EOS)) { *aSendEOS = true; } return NS_OK; }
void AudioNodeExternalInputStream::ProcessInput(GraphTime aFrom, GraphTime aTo, uint32_t aFlags) { // According to spec, number of outputs is always 1. mLastChunks.SetLength(1); // GC stuff can result in our input stream being destroyed before this stream. // Handle that. if (mInputs.IsEmpty()) { mLastChunks[0].SetNull(WEBAUDIO_BLOCK_SIZE); AdvanceOutputSegment(); return; } MOZ_ASSERT(mInputs.Length() == 1); MediaStream* source = mInputs[0]->GetSource(); nsAutoTArray<AudioSegment,1> audioSegments; nsAutoTArray<bool,1> trackMapEntriesUsed; uint32_t inputChannels = 0; for (StreamBuffer::TrackIter tracks(source->mBuffer, MediaSegment::AUDIO); !tracks.IsEnded(); tracks.Next()) { const StreamBuffer::Track& inputTrack = *tracks; // Create a TrackMapEntry if necessary. size_t trackMapIndex = GetTrackMapEntry(inputTrack, aFrom); // Maybe there's nothing in this track yet. If so, ignore it. (While the // track is only playing silence, we may not be able to determine the // correct number of channels to start resampling.) if (trackMapIndex == nsTArray<TrackMapEntry>::NoIndex) { continue; } while (trackMapEntriesUsed.Length() <= trackMapIndex) { trackMapEntriesUsed.AppendElement(false); } trackMapEntriesUsed[trackMapIndex] = true; TrackMapEntry* trackMap = &mTrackMap[trackMapIndex]; AudioSegment segment; GraphTime next; TrackRate inputTrackRate = inputTrack.GetRate(); for (GraphTime t = aFrom; t < aTo; t = next) { MediaInputPort::InputInterval interval = mInputs[0]->GetNextInputInterval(t); interval.mEnd = std::min(interval.mEnd, aTo); if (interval.mStart >= interval.mEnd) break; next = interval.mEnd; // Ticks >= startTicks and < endTicks are in the interval StreamTime outputEnd = GraphTimeToStreamTime(interval.mEnd); TrackTicks startTicks = trackMap->mSamplesPassedToResampler + segment.GetDuration(); StreamTime outputStart = GraphTimeToStreamTime(interval.mStart); NS_ASSERTION(startTicks == TimeToTicksRoundUp(inputTrackRate, outputStart), "Samples missing"); TrackTicks endTicks = TimeToTicksRoundUp(inputTrackRate, outputEnd); TrackTicks ticks = endTicks - startTicks; if (interval.mInputIsBlocked) { segment.AppendNullData(ticks); } else { // See comments in TrackUnionStream::CopyTrackData StreamTime inputStart = source->GraphTimeToStreamTime(interval.mStart); StreamTime inputEnd = source->GraphTimeToStreamTime(interval.mEnd); TrackTicks inputTrackEndPoint = inputTrack.IsEnded() ? inputTrack.GetEnd() : TRACK_TICKS_MAX; if (trackMap->mEndOfLastInputIntervalInInputStream != inputStart || trackMap->mEndOfLastInputIntervalInOutputStream != outputStart) { // Start of a new series of intervals where neither stream is blocked. trackMap->mEndOfConsumedInputTicks = TimeToTicksRoundDown(inputTrackRate, inputStart) - 1; } TrackTicks inputStartTicks = trackMap->mEndOfConsumedInputTicks; TrackTicks inputEndTicks = inputStartTicks + ticks; trackMap->mEndOfConsumedInputTicks = inputEndTicks; trackMap->mEndOfLastInputIntervalInInputStream = inputEnd; trackMap->mEndOfLastInputIntervalInOutputStream = outputEnd; if (inputStartTicks < 0) { // Data before the start of the track is just null. segment.AppendNullData(-inputStartTicks); inputStartTicks = 0; } if (inputEndTicks > inputStartTicks) { segment.AppendSlice(*inputTrack.GetSegment(), std::min(inputTrackEndPoint, inputStartTicks), std::min(inputTrackEndPoint, inputEndTicks)); } // Pad if we're looking past the end of the track segment.AppendNullData(ticks - segment.GetDuration()); } } trackMap->mSamplesPassedToResampler += segment.GetDuration(); trackMap->ResampleInputData(&segment); if (trackMap->mResampledData.GetDuration() < mCurrentOutputPosition + WEBAUDIO_BLOCK_SIZE) { // We don't have enough data. Delay it. trackMap->mResampledData.InsertNullDataAtStart( mCurrentOutputPosition + WEBAUDIO_BLOCK_SIZE - trackMap->mResampledData.GetDuration()); } audioSegments.AppendElement()->AppendSlice(trackMap->mResampledData, mCurrentOutputPosition, mCurrentOutputPosition + WEBAUDIO_BLOCK_SIZE); trackMap->mResampledData.ForgetUpTo(mCurrentOutputPosition + WEBAUDIO_BLOCK_SIZE); inputChannels = GetAudioChannelsSuperset(inputChannels, trackMap->mResamplerChannelCount); } for (int32_t i = mTrackMap.Length() - 1; i >= 0; --i) { if (i >= int32_t(trackMapEntriesUsed.Length()) || !trackMapEntriesUsed[i]) { mTrackMap.RemoveElementAt(i); } } uint32_t accumulateIndex = 0; if (inputChannels) { nsAutoTArray<float,GUESS_AUDIO_CHANNELS*WEBAUDIO_BLOCK_SIZE> downmixBuffer; for (uint32_t i = 0; i < audioSegments.Length(); ++i) { AudioChunk tmpChunk; ConvertSegmentToAudioBlock(&audioSegments[i], &tmpChunk); if (!tmpChunk.IsNull()) { if (accumulateIndex == 0) { AllocateAudioBlock(inputChannels, &mLastChunks[0]); } AccumulateInputChunk(accumulateIndex, tmpChunk, &mLastChunks[0], &downmixBuffer); accumulateIndex++; } } } if (accumulateIndex == 0) { mLastChunks[0].SetNull(WEBAUDIO_BLOCK_SIZE); } mCurrentOutputPosition += WEBAUDIO_BLOCK_SIZE; // Using AudioNodeStream's AdvanceOutputSegment to push the media stream graph along with null data. AdvanceOutputSegment(); }