void DelayBuffer::UpdateUpmixChannels(int aNewReadChunk, uint32_t aChannelCount, ChannelInterpretation aChannelInterpretation) { if (aNewReadChunk == mLastReadChunk) { MOZ_ASSERT(mUpmixChannels.Length() == aChannelCount); return; } static const float silenceChannel[WEBAUDIO_BLOCK_SIZE] = {}; NS_WARN_IF_FALSE(mHaveWrittenBlock || aNewReadChunk != mCurrentChunk, "Smoothing is making feedback delay too small."); mLastReadChunk = aNewReadChunk; // Missing assignment operator is bug 976927 mUpmixChannels.ReplaceElementsAt(0, mUpmixChannels.Length(), mChunks[aNewReadChunk].mChannelData); MOZ_ASSERT(mUpmixChannels.Length() <= aChannelCount); if (mUpmixChannels.Length() < aChannelCount) { if (aChannelInterpretation == ChannelInterpretation::Speakers) { AudioChannelsUpMix(&mUpmixChannels, aChannelCount, silenceChannel); MOZ_ASSERT(mUpmixChannels.Length() == aChannelCount, "We called GetAudioChannelsSuperset to avoid this"); } else { // Fill up the remaining channels with zeros for (uint32_t channel = mUpmixChannels.Length(); channel < aChannelCount; ++channel) { mUpmixChannels.AppendElement(silenceChannel); } } } }
static void CopyChunkToBlock(AudioChunk& aInput, AudioBlock *aBlock, uint32_t aOffsetInBlock) { uint32_t blockChannels = aBlock->ChannelCount(); AutoTArray<const T*,2> channels; if (aInput.IsNull()) { channels.SetLength(blockChannels); PodZero(channels.Elements(), blockChannels); } else { const nsTArray<const T*>& inputChannels = aInput.ChannelData<T>(); channels.SetLength(inputChannels.Length()); PodCopy(channels.Elements(), inputChannels.Elements(), channels.Length()); if (channels.Length() != blockChannels) { // We only need to upmix here because aBlock's channel count has been // chosen to be a superset of the channel count of every chunk. AudioChannelsUpMix(&channels, blockChannels, static_cast<T*>(nullptr)); } } for (uint32_t c = 0; c < blockChannels; ++c) { float* outputData = aBlock->ChannelFloatsForWrite(c) + aOffsetInBlock; if (channels[c]) { ConvertAudioSamplesWithScale(channels[c], outputData, aInput.GetDuration(), aInput.mVolume); } else { PodZero(outputData, aInput.GetDuration()); } } }
void DelayBuffer::UpdateUpmixChannels(int aNewReadChunk, uint32_t aChannelCount, ChannelInterpretation aChannelInterpretation) { if (aNewReadChunk == mLastReadChunk) { MOZ_ASSERT(mUpmixChannels.Length() == aChannelCount); return; } NS_WARNING_ASSERTION(mHaveWrittenBlock || aNewReadChunk != mCurrentChunk, "Smoothing is making feedback delay too small."); mLastReadChunk = aNewReadChunk; mUpmixChannels = mChunks[aNewReadChunk].ChannelData<float>(); MOZ_ASSERT(mUpmixChannels.Length() <= aChannelCount); if (mUpmixChannels.Length() < aChannelCount) { if (aChannelInterpretation == ChannelInterpretation::Speakers) { AudioChannelsUpMix(&mUpmixChannels, aChannelCount, SilentChannel::ZeroChannel<float>()); MOZ_ASSERT(mUpmixChannels.Length() == aChannelCount, "We called GetAudioChannelsSuperset to avoid this"); } else { // Fill up the remaining channels with zeros for (uint32_t channel = mUpmixChannels.Length(); channel < aChannelCount; ++channel) { mUpmixChannels.AppendElement(SilentChannel::ZeroChannel<float>()); } } } }
void AudioNodeStream::UpMixDownMixChunk(const AudioChunk* aChunk, uint32_t aOutputChannelCount, nsTArray<const void*>& aOutputChannels, nsTArray<float>& aDownmixBuffer) { static const float silenceChannel[WEBAUDIO_BLOCK_SIZE] = {0.f}; aOutputChannels.AppendElements(aChunk->mChannelData); if (aOutputChannels.Length() < aOutputChannelCount) { if (mChannelInterpretation == ChannelInterpretation::Speakers) { AudioChannelsUpMix(&aOutputChannels, aOutputChannelCount, nullptr); NS_ASSERTION(aOutputChannelCount == aOutputChannels.Length(), "We called GetAudioChannelsSuperset to avoid this"); } else { // Fill up the remaining aOutputChannels by zeros for (uint32_t j = aOutputChannels.Length(); j < aOutputChannelCount; ++j) { aOutputChannels.AppendElement(silenceChannel); } } } else if (aOutputChannels.Length() > aOutputChannelCount) { if (mChannelInterpretation == ChannelInterpretation::Speakers) { nsAutoTArray<float*,GUESS_AUDIO_CHANNELS> outputChannels; outputChannels.SetLength(aOutputChannelCount); aDownmixBuffer.SetLength(aOutputChannelCount * WEBAUDIO_BLOCK_SIZE); for (uint32_t j = 0; j < aOutputChannelCount; ++j) { outputChannels[j] = &aDownmixBuffer[j * WEBAUDIO_BLOCK_SIZE]; } AudioChannelsDownMix(aOutputChannels, outputChannels.Elements(), aOutputChannelCount, WEBAUDIO_BLOCK_SIZE); aOutputChannels.SetLength(aOutputChannelCount); for (uint32_t j = 0; j < aOutputChannels.Length(); ++j) { aOutputChannels[j] = outputChannels[j]; } } else { // Drop the remaining aOutputChannels aOutputChannels.RemoveElementsAt(aOutputChannelCount, aOutputChannels.Length() - aOutputChannelCount); } } }
/** * Copies the data in aInput to aOffsetInBlock within aBlock. * aBlock must have been allocated with AllocateInputBlock and have a channel * count that's a superset of the channels in aInput. */ static void CopyChunkToBlock(const AudioChunk& aInput, AudioChunk *aBlock, uint32_t aOffsetInBlock) { uint32_t blockChannels = aBlock->ChannelCount(); nsAutoTArray<const void*,2> channels; if (aInput.IsNull()) { channels.SetLength(blockChannels); PodZero(channels.Elements(), blockChannels); } else { channels.SetLength(aInput.ChannelCount()); PodCopy(channels.Elements(), aInput.mChannelData.Elements(), channels.Length()); if (channels.Length() != blockChannels) { // We only need to upmix here because aBlock's channel count has been // chosen to be a superset of the channel count of every chunk. AudioChannelsUpMix(&channels, blockChannels, nullptr); } } uint32_t duration = aInput.GetDuration(); for (uint32_t c = 0; c < blockChannels; ++c) { float* outputData = static_cast<float*>(const_cast<void*>(aBlock->mChannelData[c])) + aOffsetInBlock; if (channels[c]) { switch (aInput.mBufferFormat) { case AUDIO_FORMAT_FLOAT32: ConvertAudioSamplesWithScale( static_cast<const float*>(channels[c]), outputData, duration, aInput.mVolume); break; case AUDIO_FORMAT_S16: ConvertAudioSamplesWithScale( static_cast<const int16_t*>(channels[c]), outputData, duration, aInput.mVolume); break; default: NS_ERROR("Unhandled format"); } } else { PodZero(outputData, duration); } } }
void AudioTrackEncoder::InterleaveTrackData(AudioChunk& aChunk, int32_t aDuration, uint32_t aOutputChannels, AudioDataValue* aOutput) { if (aChunk.mChannelData.Length() < aOutputChannels) { // Up-mix. This might make the mChannelData have more than aChannels. AudioChannelsUpMix(&aChunk.mChannelData, aOutputChannels, gZeroChannel); } if (aChunk.mChannelData.Length() > aOutputChannels) { DownmixAndInterleave(aChunk.mChannelData, aChunk.mBufferFormat, aDuration, aChunk.mVolume, mChannels, aOutput); } else { InterleaveAndConvertBuffer(aChunk.mChannelData.Elements(), aChunk.mBufferFormat, aDuration, aChunk.mVolume, mChannels, aOutput); } }
void TestUpmixStereo() { size_t arraySize = 1024; nsTArray<T*> channels; nsTArray<const T*> channelsptr; channels.SetLength(1); channelsptr.SetLength(1); channels[0] = new T[arraySize]; for (size_t i = 0; i < arraySize; i++) { channels[0][i] = GetHighValue<T>(); } channelsptr[0] = channels[0]; AudioChannelsUpMix(&channelsptr, 2, SilentChannel<T>()); for (size_t channel = 0; channel < 2; channel++) { for (size_t i = 0; i < arraySize; i++) { ASSERT_TRUE(channelsptr[channel][i] == GetHighValue<T>()); } } delete[] channels[0]; }
AudioChunk* AudioNodeStream::ObtainInputBlock(AudioChunk* aTmpChunk) { uint32_t inputCount = mInputs.Length(); uint32_t outputChannelCount = 0; nsAutoTArray<AudioChunk*,250> inputChunks; for (uint32_t i = 0; i < inputCount; ++i) { MediaStream* s = mInputs[i]->GetSource(); AudioNodeStream* a = static_cast<AudioNodeStream*>(s); MOZ_ASSERT(a == s->AsAudioNodeStream()); if (a->IsFinishedOnGraphThread()) { continue; } AudioChunk* chunk = &a->mLastChunk; // XXX when we implement DelayNode, this will no longer be true and we'll // need to treat a null chunk (when the DelayNode hasn't had a chance // to produce data yet) as silence here. MOZ_ASSERT(chunk); if (chunk->IsNull()) { continue; } inputChunks.AppendElement(chunk); outputChannelCount = GetAudioChannelsSuperset(outputChannelCount, chunk->mChannelData.Length()); } uint32_t inputChunkCount = inputChunks.Length(); if (inputChunkCount == 0) { aTmpChunk->SetNull(WEBAUDIO_BLOCK_SIZE); return aTmpChunk; } if (inputChunkCount == 1) { return inputChunks[0]; } AllocateAudioBlock(outputChannelCount, aTmpChunk); for (uint32_t i = 0; i < inputChunkCount; ++i) { AudioChunk* chunk = inputChunks[i]; nsAutoTArray<const void*,GUESS_AUDIO_CHANNELS> channels; channels.AppendElements(chunk->mChannelData); if (channels.Length() < outputChannelCount) { AudioChannelsUpMix(&channels, outputChannelCount, nullptr); NS_ASSERTION(outputChannelCount == channels.Length(), "We called GetAudioChannelsSuperset to avoid this"); } for (uint32_t c = 0; c < channels.Length(); ++c) { const float* inputData = static_cast<const float*>(channels[c]); float* outputData = static_cast<float*>(const_cast<void*>(aTmpChunk->mChannelData[c])); if (inputData) { if (i == 0) { AudioBlockCopyChannelWithScale(inputData, chunk->mVolume, outputData); } else { AudioBlockAddChannelWithScale(inputData, chunk->mVolume, outputData); } } else { if (i == 0) { memset(outputData, 0, WEBAUDIO_BLOCK_SIZE*sizeof(float)); } } } } return aTmpChunk; }
void AudioNodeStream::ObtainInputBlock(AudioChunk& aTmpChunk, uint32_t aPortIndex) { uint32_t inputCount = mInputs.Length(); uint32_t outputChannelCount = 1; nsAutoTArray<AudioChunk*,250> inputChunks; for (uint32_t i = 0; i < inputCount; ++i) { if (aPortIndex != mInputs[i]->InputNumber()) { // This input is connected to a different port continue; } MediaStream* s = mInputs[i]->GetSource(); AudioNodeStream* a = static_cast<AudioNodeStream*>(s); MOZ_ASSERT(a == s->AsAudioNodeStream()); if (a->IsFinishedOnGraphThread() || a->IsAudioParamStream()) { continue; } AudioChunk* chunk = &a->mLastChunks[mInputs[i]->OutputNumber()]; MOZ_ASSERT(chunk); if (chunk->IsNull()) { continue; } inputChunks.AppendElement(chunk); outputChannelCount = GetAudioChannelsSuperset(outputChannelCount, chunk->mChannelData.Length()); } switch (mChannelCountMode) { case ChannelCountMode::Explicit: // Disregard the output channel count that we've calculated, and just use // mNumberOfInputChannels. outputChannelCount = mNumberOfInputChannels; break; case ChannelCountMode::Clamped_max: // Clamp the computed output channel count to mNumberOfInputChannels. outputChannelCount = std::min(outputChannelCount, mNumberOfInputChannels); break; case ChannelCountMode::Max: // Nothing to do here, just shut up the compiler warning. break; } uint32_t inputChunkCount = inputChunks.Length(); if (inputChunkCount == 0 || (inputChunkCount == 1 && inputChunks[0]->mChannelData.Length() == 0)) { aTmpChunk.SetNull(WEBAUDIO_BLOCK_SIZE); return; } if (inputChunkCount == 1 && inputChunks[0]->mChannelData.Length() == outputChannelCount) { aTmpChunk = *inputChunks[0]; return; } AllocateAudioBlock(outputChannelCount, &aTmpChunk); float silenceChannel[WEBAUDIO_BLOCK_SIZE] = {0.f}; // The static storage here should be 1KB, so it's fine nsAutoTArray<float, GUESS_AUDIO_CHANNELS*WEBAUDIO_BLOCK_SIZE> downmixBuffer; for (uint32_t i = 0; i < inputChunkCount; ++i) { AudioChunk* chunk = inputChunks[i]; nsAutoTArray<const void*,GUESS_AUDIO_CHANNELS> channels; channels.AppendElements(chunk->mChannelData); if (channels.Length() < outputChannelCount) { if (mChannelInterpretation == ChannelInterpretation::Speakers) { AudioChannelsUpMix(&channels, outputChannelCount, nullptr); NS_ASSERTION(outputChannelCount == channels.Length(), "We called GetAudioChannelsSuperset to avoid this"); } else { // Fill up the remaining channels by zeros for (uint32_t j = channels.Length(); j < outputChannelCount; ++j) { channels.AppendElement(silenceChannel); } } } else if (channels.Length() > outputChannelCount) { if (mChannelInterpretation == ChannelInterpretation::Speakers) { nsAutoTArray<float*,GUESS_AUDIO_CHANNELS> outputChannels; outputChannels.SetLength(outputChannelCount); downmixBuffer.SetLength(outputChannelCount * WEBAUDIO_BLOCK_SIZE); for (uint32_t j = 0; j < outputChannelCount; ++j) { outputChannels[j] = &downmixBuffer[j * WEBAUDIO_BLOCK_SIZE]; } AudioChannelsDownMix(channels, outputChannels.Elements(), outputChannelCount, WEBAUDIO_BLOCK_SIZE); channels.SetLength(outputChannelCount); for (uint32_t j = 0; j < channels.Length(); ++j) { channels[j] = outputChannels[j]; } } else { // Drop the remaining channels channels.RemoveElementsAt(outputChannelCount, channels.Length() - outputChannelCount); } } for (uint32_t c = 0; c < channels.Length(); ++c) { const float* inputData = static_cast<const float*>(channels[c]); float* outputData = static_cast<float*>(const_cast<void*>(aTmpChunk.mChannelData[c])); if (inputData) { if (i == 0) { AudioBlockCopyChannelWithScale(inputData, chunk->mVolume, outputData); } else { AudioBlockAddChannelWithScale(inputData, chunk->mVolume, outputData); } } else { if (i == 0) { memset(outputData, 0, WEBAUDIO_BLOCK_SIZE*sizeof(float)); } } } } }
void AudioSegment::WriteTo(uint64_t aID, AudioStream* aOutput, AudioMixer* aMixer) { uint32_t outputChannels = aOutput->GetChannels(); nsAutoTArray<AudioDataValue,AUDIO_PROCESSING_FRAMES*GUESS_AUDIO_CHANNELS> buf; nsAutoTArray<const void*,GUESS_AUDIO_CHANNELS> channelData; // Offset in the buffer that will end up sent to the AudioStream, in samples. uint32_t offset = 0; if (!GetDuration()) { return; } uint32_t outBufferLength = GetDuration() * outputChannels; buf.SetLength(outBufferLength); for (ChunkIterator ci(*this); !ci.IsEnded(); ci.Next()) { AudioChunk& c = *ci; uint32_t frames = c.mDuration; // If we have written data in the past, or we have real (non-silent) data // to write, we can proceed. Otherwise, it means we just started the // AudioStream, and we don't have real data to write to it (just silence). // To avoid overbuffering in the AudioStream, we simply drop the silence, // here. The stream will underrun and output silence anyways. if (c.mBuffer || aOutput->GetWritten()) { if (c.mBuffer && c.mBufferFormat != AUDIO_FORMAT_SILENCE) { channelData.SetLength(c.mChannelData.Length()); for (uint32_t i = 0; i < channelData.Length(); ++i) { channelData[i] = c.mChannelData[i]; } if (channelData.Length() < outputChannels) { // Up-mix. Note that this might actually make channelData have more // than outputChannels temporarily. AudioChannelsUpMix(&channelData, outputChannels, gZeroChannel); } if (channelData.Length() > outputChannels) { // Down-mix. DownmixAndInterleave(channelData, c.mBufferFormat, frames, c.mVolume, outputChannels, buf.Elements() + offset); } else { InterleaveAndConvertBuffer(channelData.Elements(), c.mBufferFormat, frames, c.mVolume, outputChannels, buf.Elements() + offset); } } else { // Assumes that a bit pattern of zeroes == 0.0f memset(buf.Elements() + offset, 0, outputChannels * frames * sizeof(AudioDataValue)); } offset += frames * outputChannels; } if (!c.mTimeStamp.IsNull()) { TimeStamp now = TimeStamp::Now(); // would be more efficient to c.mTimeStamp to ms on create time then pass here LogTime(AsyncLatencyLogger::AudioMediaStreamTrack, aID, (now - c.mTimeStamp).ToMilliseconds(), c.mTimeStamp); } } aOutput->Write(buf.Elements(), offset / outputChannels, &(mChunks[mChunks.Length() - 1].mTimeStamp)); if (aMixer) { aMixer->Mix(buf.Elements(), outputChannels, GetDuration(), aOutput->GetRate()); } aOutput->Start(); }
void AudioNodeExternalInputStream::TrackMapEntry::ResampleInputData(AudioSegment* aSegment) { AudioSegment::ChunkIterator ci(*aSegment); while (!ci.IsEnded()) { const AudioChunk& chunk = *ci; nsAutoTArray<const void*,2> channels; if (chunk.GetDuration() > UINT32_MAX) { // This will cause us to OOM or overflow below. So let's just bail. NS_ERROR("Chunk duration out of bounds"); return; } uint32_t duration = uint32_t(chunk.GetDuration()); if (chunk.IsNull()) { nsAutoTArray<AudioDataValue,1024> silence; silence.SetLength(duration); PodZero(silence.Elements(), silence.Length()); channels.SetLength(mResamplerChannelCount); for (uint32_t i = 0; i < channels.Length(); ++i) { channels[i] = silence.Elements(); } ResampleChannels(channels, duration, AUDIO_OUTPUT_FORMAT, 0.0f); } else if (chunk.mChannelData.Length() == mResamplerChannelCount) { // Common case, since mResamplerChannelCount is set to the first chunk's // number of channels. channels.AppendElements(chunk.mChannelData); ResampleChannels(channels, duration, chunk.mBufferFormat, chunk.mVolume); } else { // Uncommon case. Since downmixing requires channels to be floats, // convert everything to floats now. uint32_t upChannels = GetAudioChannelsSuperset(chunk.mChannelData.Length(), mResamplerChannelCount); nsTArray<float> buffer; if (chunk.mBufferFormat == AUDIO_FORMAT_FLOAT32) { channels.AppendElements(chunk.mChannelData); } else { NS_ASSERTION(chunk.mBufferFormat == AUDIO_FORMAT_S16, "Unknown format"); if (duration > UINT32_MAX/chunk.mChannelData.Length()) { NS_ERROR("Chunk duration out of bounds"); return; } buffer.SetLength(chunk.mChannelData.Length()*duration); for (uint32_t i = 0; i < chunk.mChannelData.Length(); ++i) { const int16_t* samples = static_cast<const int16_t*>(chunk.mChannelData[i]); float* converted = &buffer[i*duration]; for (uint32_t j = 0; j < duration; ++j) { converted[j] = AudioSampleToFloat(samples[j]); } channels.AppendElement(converted); } } nsTArray<float> zeroes; if (channels.Length() < upChannels) { zeroes.SetLength(duration); PodZero(zeroes.Elements(), zeroes.Length()); AudioChannelsUpMix(&channels, upChannels, zeroes.Elements()); } if (channels.Length() == mResamplerChannelCount) { ResampleChannels(channels, duration, AUDIO_FORMAT_FLOAT32, chunk.mVolume); } else { nsTArray<float> output; if (duration > UINT32_MAX/mResamplerChannelCount) { NS_ERROR("Chunk duration out of bounds"); return; } output.SetLength(duration*mResamplerChannelCount); nsAutoTArray<float*,2> outputPtrs; nsAutoTArray<const void*,2> outputPtrsConst; for (uint32_t i = 0; i < mResamplerChannelCount; ++i) { outputPtrs.AppendElement(output.Elements() + i*duration); outputPtrsConst.AppendElement(outputPtrs[i]); } AudioChannelsDownMix(channels, outputPtrs.Elements(), outputPtrs.Length(), duration); ResampleChannels(outputPtrsConst, duration, AUDIO_FORMAT_FLOAT32, chunk.mVolume); } } ci.Next(); } }