void
DelayBuffer::UpdateUpmixChannels(int aNewReadChunk, uint32_t aChannelCount,
ChannelInterpretation aChannelInterpretation)
{
if (aNewReadChunk == mLastReadChunk) {
MOZ_ASSERT(mUpmixChannels.Length() == aChannelCount);
return;
}
static const float silenceChannel[WEBAUDIO_BLOCK_SIZE] = {};
NS_WARN_IF_FALSE(mHaveWrittenBlock || aNewReadChunk != mCurrentChunk,
"Smoothing is making feedback delay too small.");
mLastReadChunk = aNewReadChunk;
// Missing assignment operator is bug 976927
mUpmixChannels.ReplaceElementsAt(0, mUpmixChannels.Length(),
mChunks[aNewReadChunk].mChannelData);
MOZ_ASSERT(mUpmixChannels.Length() <= aChannelCount);
if (mUpmixChannels.Length() < aChannelCount) {
if (aChannelInterpretation == ChannelInterpretation::Speakers) {
AudioChannelsUpMix(&mUpmixChannels, aChannelCount, silenceChannel);
MOZ_ASSERT(mUpmixChannels.Length() == aChannelCount,
"We called GetAudioChannelsSuperset to avoid this");
} else {
// Fill up the remaining channels with zeros
for (uint32_t channel = mUpmixChannels.Length();
channel < aChannelCount; ++channel) {
mUpmixChannels.AppendElement(silenceChannel);
}
}
}
}
static void
CopyChunkToBlock(AudioChunk& aInput, AudioBlock *aBlock,
uint32_t aOffsetInBlock)
{
uint32_t blockChannels = aBlock->ChannelCount();
AutoTArray<const T*,2> channels;
if (aInput.IsNull()) {
channels.SetLength(blockChannels);
PodZero(channels.Elements(), blockChannels);
} else {
const nsTArray<const T*>& inputChannels = aInput.ChannelData<T>();
channels.SetLength(inputChannels.Length());
PodCopy(channels.Elements(), inputChannels.Elements(), channels.Length());
if (channels.Length() != blockChannels) {
// We only need to upmix here because aBlock's channel count has been
// chosen to be a superset of the channel count of every chunk.
AudioChannelsUpMix(&channels, blockChannels, static_cast<T*>(nullptr));
}
}
for (uint32_t c = 0; c < blockChannels; ++c) {
float* outputData = aBlock->ChannelFloatsForWrite(c) + aOffsetInBlock;
if (channels[c]) {
ConvertAudioSamplesWithScale(channels[c], outputData, aInput.GetDuration(), aInput.mVolume);
} else {
PodZero(outputData, aInput.GetDuration());
}
}
}
void
DelayBuffer::UpdateUpmixChannels(int aNewReadChunk, uint32_t aChannelCount,
ChannelInterpretation aChannelInterpretation)
{
if (aNewReadChunk == mLastReadChunk) {
MOZ_ASSERT(mUpmixChannels.Length() == aChannelCount);
return;
}
NS_WARNING_ASSERTION(mHaveWrittenBlock || aNewReadChunk != mCurrentChunk,
"Smoothing is making feedback delay too small.");
mLastReadChunk = aNewReadChunk;
mUpmixChannels = mChunks[aNewReadChunk].ChannelData<float>();
MOZ_ASSERT(mUpmixChannels.Length() <= aChannelCount);
if (mUpmixChannels.Length() < aChannelCount) {
if (aChannelInterpretation == ChannelInterpretation::Speakers) {
AudioChannelsUpMix(&mUpmixChannels,
aChannelCount, SilentChannel::ZeroChannel<float>());
MOZ_ASSERT(mUpmixChannels.Length() == aChannelCount,
"We called GetAudioChannelsSuperset to avoid this");
} else {
// Fill up the remaining channels with zeros
for (uint32_t channel = mUpmixChannels.Length();
channel < aChannelCount; ++channel) {
mUpmixChannels.AppendElement(SilentChannel::ZeroChannel<float>());
}
}
}
}
void
AudioNodeStream::UpMixDownMixChunk(const AudioChunk* aChunk,
uint32_t aOutputChannelCount,
nsTArray<const void*>& aOutputChannels,
nsTArray<float>& aDownmixBuffer)
{
static const float silenceChannel[WEBAUDIO_BLOCK_SIZE] = {0.f};
aOutputChannels.AppendElements(aChunk->mChannelData);
if (aOutputChannels.Length() < aOutputChannelCount) {
if (mChannelInterpretation == ChannelInterpretation::Speakers) {
AudioChannelsUpMix(&aOutputChannels, aOutputChannelCount, nullptr);
NS_ASSERTION(aOutputChannelCount == aOutputChannels.Length(),
"We called GetAudioChannelsSuperset to avoid this");
} else {
// Fill up the remaining aOutputChannels by zeros
for (uint32_t j = aOutputChannels.Length(); j < aOutputChannelCount; ++j) {
aOutputChannels.AppendElement(silenceChannel);
}
}
} else if (aOutputChannels.Length() > aOutputChannelCount) {
if (mChannelInterpretation == ChannelInterpretation::Speakers) {
nsAutoTArray<float*,GUESS_AUDIO_CHANNELS> outputChannels;
outputChannels.SetLength(aOutputChannelCount);
aDownmixBuffer.SetLength(aOutputChannelCount * WEBAUDIO_BLOCK_SIZE);
for (uint32_t j = 0; j < aOutputChannelCount; ++j) {
outputChannels[j] = &aDownmixBuffer[j * WEBAUDIO_BLOCK_SIZE];
}
AudioChannelsDownMix(aOutputChannels, outputChannels.Elements(),
aOutputChannelCount, WEBAUDIO_BLOCK_SIZE);
aOutputChannels.SetLength(aOutputChannelCount);
for (uint32_t j = 0; j < aOutputChannels.Length(); ++j) {
aOutputChannels[j] = outputChannels[j];
}
} else {
// Drop the remaining aOutputChannels
aOutputChannels.RemoveElementsAt(aOutputChannelCount,
aOutputChannels.Length() - aOutputChannelCount);
}
}
}
/**
* Copies the data in aInput to aOffsetInBlock within aBlock.
* aBlock must have been allocated with AllocateInputBlock and have a channel
* count that's a superset of the channels in aInput.
*/
static void
CopyChunkToBlock(const AudioChunk& aInput, AudioChunk *aBlock,
uint32_t aOffsetInBlock)
{
uint32_t blockChannels = aBlock->ChannelCount();
nsAutoTArray<const void*,2> channels;
if (aInput.IsNull()) {
channels.SetLength(blockChannels);
PodZero(channels.Elements(), blockChannels);
} else {
channels.SetLength(aInput.ChannelCount());
PodCopy(channels.Elements(), aInput.mChannelData.Elements(), channels.Length());
if (channels.Length() != blockChannels) {
// We only need to upmix here because aBlock's channel count has been
// chosen to be a superset of the channel count of every chunk.
AudioChannelsUpMix(&channels, blockChannels, nullptr);
}
}
uint32_t duration = aInput.GetDuration();
for (uint32_t c = 0; c < blockChannels; ++c) {
float* outputData =
static_cast<float*>(const_cast<void*>(aBlock->mChannelData[c])) + aOffsetInBlock;
if (channels[c]) {
switch (aInput.mBufferFormat) {
case AUDIO_FORMAT_FLOAT32:
ConvertAudioSamplesWithScale(
static_cast<const float*>(channels[c]), outputData, duration,
aInput.mVolume);
break;
case AUDIO_FORMAT_S16:
ConvertAudioSamplesWithScale(
static_cast<const int16_t*>(channels[c]), outputData, duration,
aInput.mVolume);
break;
default:
NS_ERROR("Unhandled format");
}
} else {
PodZero(outputData, duration);
}
}
}
void
AudioTrackEncoder::InterleaveTrackData(AudioChunk& aChunk,
int32_t aDuration,
uint32_t aOutputChannels,
AudioDataValue* aOutput)
{
if (aChunk.mChannelData.Length() < aOutputChannels) {
// Up-mix. This might make the mChannelData have more than aChannels.
AudioChannelsUpMix(&aChunk.mChannelData, aOutputChannels, gZeroChannel);
}
if (aChunk.mChannelData.Length() > aOutputChannels) {
DownmixAndInterleave(aChunk.mChannelData, aChunk.mBufferFormat, aDuration,
aChunk.mVolume, mChannels, aOutput);
} else {
InterleaveAndConvertBuffer(aChunk.mChannelData.Elements(),
aChunk.mBufferFormat, aDuration, aChunk.mVolume,
mChannels, aOutput);
}
}
void TestUpmixStereo() {
size_t arraySize = 1024;
nsTArray<T*> channels;
nsTArray<const T*> channelsptr;
channels.SetLength(1);
channelsptr.SetLength(1);
channels[0] = new T[arraySize];
for (size_t i = 0; i < arraySize; i++) {
channels[0][i] = GetHighValue<T>();
}
channelsptr[0] = channels[0];
AudioChannelsUpMix(&channelsptr, 2, SilentChannel<T>());
for (size_t channel = 0; channel < 2; channel++) {
for (size_t i = 0; i < arraySize; i++) {
ASSERT_TRUE(channelsptr[channel][i] == GetHighValue<T>());
}
}
delete[] channels[0];
}
AudioChunk*
AudioNodeStream::ObtainInputBlock(AudioChunk* aTmpChunk)
{
uint32_t inputCount = mInputs.Length();
uint32_t outputChannelCount = 0;
nsAutoTArray<AudioChunk*,250> inputChunks;
for (uint32_t i = 0; i < inputCount; ++i) {
MediaStream* s = mInputs[i]->GetSource();
AudioNodeStream* a = static_cast<AudioNodeStream*>(s);
MOZ_ASSERT(a == s->AsAudioNodeStream());
if (a->IsFinishedOnGraphThread()) {
continue;
}
AudioChunk* chunk = &a->mLastChunk;
// XXX when we implement DelayNode, this will no longer be true and we'll
// need to treat a null chunk (when the DelayNode hasn't had a chance
// to produce data yet) as silence here.
MOZ_ASSERT(chunk);
if (chunk->IsNull()) {
continue;
}
inputChunks.AppendElement(chunk);
outputChannelCount =
GetAudioChannelsSuperset(outputChannelCount, chunk->mChannelData.Length());
}
uint32_t inputChunkCount = inputChunks.Length();
if (inputChunkCount == 0) {
aTmpChunk->SetNull(WEBAUDIO_BLOCK_SIZE);
return aTmpChunk;
}
if (inputChunkCount == 1) {
return inputChunks[0];
}
AllocateAudioBlock(outputChannelCount, aTmpChunk);
for (uint32_t i = 0; i < inputChunkCount; ++i) {
AudioChunk* chunk = inputChunks[i];
nsAutoTArray<const void*,GUESS_AUDIO_CHANNELS> channels;
channels.AppendElements(chunk->mChannelData);
if (channels.Length() < outputChannelCount) {
AudioChannelsUpMix(&channels, outputChannelCount, nullptr);
NS_ASSERTION(outputChannelCount == channels.Length(),
"We called GetAudioChannelsSuperset to avoid this");
}
for (uint32_t c = 0; c < channels.Length(); ++c) {
const float* inputData = static_cast<const float*>(channels[c]);
float* outputData = static_cast<float*>(const_cast<void*>(aTmpChunk->mChannelData[c]));
if (inputData) {
if (i == 0) {
AudioBlockCopyChannelWithScale(inputData, chunk->mVolume, outputData);
} else {
AudioBlockAddChannelWithScale(inputData, chunk->mVolume, outputData);
}
} else {
if (i == 0) {
memset(outputData, 0, WEBAUDIO_BLOCK_SIZE*sizeof(float));
}
}
}
}
return aTmpChunk;
}
void
AudioNodeStream::ObtainInputBlock(AudioChunk& aTmpChunk, uint32_t aPortIndex)
{
uint32_t inputCount = mInputs.Length();
uint32_t outputChannelCount = 1;
nsAutoTArray<AudioChunk*,250> inputChunks;
for (uint32_t i = 0; i < inputCount; ++i) {
if (aPortIndex != mInputs[i]->InputNumber()) {
// This input is connected to a different port
continue;
}
MediaStream* s = mInputs[i]->GetSource();
AudioNodeStream* a = static_cast<AudioNodeStream*>(s);
MOZ_ASSERT(a == s->AsAudioNodeStream());
if (a->IsFinishedOnGraphThread() ||
a->IsAudioParamStream()) {
continue;
}
AudioChunk* chunk = &a->mLastChunks[mInputs[i]->OutputNumber()];
MOZ_ASSERT(chunk);
if (chunk->IsNull()) {
continue;
}
inputChunks.AppendElement(chunk);
outputChannelCount =
GetAudioChannelsSuperset(outputChannelCount, chunk->mChannelData.Length());
}
switch (mChannelCountMode) {
case ChannelCountMode::Explicit:
// Disregard the output channel count that we've calculated, and just use
// mNumberOfInputChannels.
outputChannelCount = mNumberOfInputChannels;
break;
case ChannelCountMode::Clamped_max:
// Clamp the computed output channel count to mNumberOfInputChannels.
outputChannelCount = std::min(outputChannelCount, mNumberOfInputChannels);
break;
case ChannelCountMode::Max:
// Nothing to do here, just shut up the compiler warning.
break;
}
uint32_t inputChunkCount = inputChunks.Length();
if (inputChunkCount == 0 ||
(inputChunkCount == 1 && inputChunks[0]->mChannelData.Length() == 0)) {
aTmpChunk.SetNull(WEBAUDIO_BLOCK_SIZE);
return;
}
if (inputChunkCount == 1 &&
inputChunks[0]->mChannelData.Length() == outputChannelCount) {
aTmpChunk = *inputChunks[0];
return;
}
AllocateAudioBlock(outputChannelCount, &aTmpChunk);
float silenceChannel[WEBAUDIO_BLOCK_SIZE] = {0.f};
// The static storage here should be 1KB, so it's fine
nsAutoTArray<float, GUESS_AUDIO_CHANNELS*WEBAUDIO_BLOCK_SIZE> downmixBuffer;
for (uint32_t i = 0; i < inputChunkCount; ++i) {
AudioChunk* chunk = inputChunks[i];
nsAutoTArray<const void*,GUESS_AUDIO_CHANNELS> channels;
channels.AppendElements(chunk->mChannelData);
if (channels.Length() < outputChannelCount) {
if (mChannelInterpretation == ChannelInterpretation::Speakers) {
AudioChannelsUpMix(&channels, outputChannelCount, nullptr);
NS_ASSERTION(outputChannelCount == channels.Length(),
"We called GetAudioChannelsSuperset to avoid this");
} else {
// Fill up the remaining channels by zeros
for (uint32_t j = channels.Length(); j < outputChannelCount; ++j) {
channels.AppendElement(silenceChannel);
}
}
} else if (channels.Length() > outputChannelCount) {
if (mChannelInterpretation == ChannelInterpretation::Speakers) {
nsAutoTArray<float*,GUESS_AUDIO_CHANNELS> outputChannels;
outputChannels.SetLength(outputChannelCount);
downmixBuffer.SetLength(outputChannelCount * WEBAUDIO_BLOCK_SIZE);
for (uint32_t j = 0; j < outputChannelCount; ++j) {
outputChannels[j] = &downmixBuffer[j * WEBAUDIO_BLOCK_SIZE];
}
AudioChannelsDownMix(channels, outputChannels.Elements(),
outputChannelCount, WEBAUDIO_BLOCK_SIZE);
channels.SetLength(outputChannelCount);
for (uint32_t j = 0; j < channels.Length(); ++j) {
channels[j] = outputChannels[j];
}
} else {
// Drop the remaining channels
channels.RemoveElementsAt(outputChannelCount,
channels.Length() - outputChannelCount);
}
}
for (uint32_t c = 0; c < channels.Length(); ++c) {
const float* inputData = static_cast<const float*>(channels[c]);
float* outputData = static_cast<float*>(const_cast<void*>(aTmpChunk.mChannelData[c]));
if (inputData) {
if (i == 0) {
AudioBlockCopyChannelWithScale(inputData, chunk->mVolume, outputData);
} else {
AudioBlockAddChannelWithScale(inputData, chunk->mVolume, outputData);
}
} else {
if (i == 0) {
memset(outputData, 0, WEBAUDIO_BLOCK_SIZE*sizeof(float));
}
}
}
}
}
void
AudioSegment::WriteTo(uint64_t aID, AudioStream* aOutput, AudioMixer* aMixer)
{
uint32_t outputChannels = aOutput->GetChannels();
nsAutoTArray<AudioDataValue,AUDIO_PROCESSING_FRAMES*GUESS_AUDIO_CHANNELS> buf;
nsAutoTArray<const void*,GUESS_AUDIO_CHANNELS> channelData;
// Offset in the buffer that will end up sent to the AudioStream, in samples.
uint32_t offset = 0;
if (!GetDuration()) {
return;
}
uint32_t outBufferLength = GetDuration() * outputChannels;
buf.SetLength(outBufferLength);
for (ChunkIterator ci(*this); !ci.IsEnded(); ci.Next()) {
AudioChunk& c = *ci;
uint32_t frames = c.mDuration;
// If we have written data in the past, or we have real (non-silent) data
// to write, we can proceed. Otherwise, it means we just started the
// AudioStream, and we don't have real data to write to it (just silence).
// To avoid overbuffering in the AudioStream, we simply drop the silence,
// here. The stream will underrun and output silence anyways.
if (c.mBuffer || aOutput->GetWritten()) {
if (c.mBuffer && c.mBufferFormat != AUDIO_FORMAT_SILENCE) {
channelData.SetLength(c.mChannelData.Length());
for (uint32_t i = 0; i < channelData.Length(); ++i) {
channelData[i] = c.mChannelData[i];
}
if (channelData.Length() < outputChannels) {
// Up-mix. Note that this might actually make channelData have more
// than outputChannels temporarily.
AudioChannelsUpMix(&channelData, outputChannels, gZeroChannel);
}
if (channelData.Length() > outputChannels) {
// Down-mix.
DownmixAndInterleave(channelData, c.mBufferFormat, frames,
c.mVolume, outputChannels, buf.Elements() + offset);
} else {
InterleaveAndConvertBuffer(channelData.Elements(), c.mBufferFormat,
frames, c.mVolume,
outputChannels,
buf.Elements() + offset);
}
} else {
// Assumes that a bit pattern of zeroes == 0.0f
memset(buf.Elements() + offset, 0, outputChannels * frames * sizeof(AudioDataValue));
}
offset += frames * outputChannels;
}
if (!c.mTimeStamp.IsNull()) {
TimeStamp now = TimeStamp::Now();
// would be more efficient to c.mTimeStamp to ms on create time then pass here
LogTime(AsyncLatencyLogger::AudioMediaStreamTrack, aID,
(now - c.mTimeStamp).ToMilliseconds(), c.mTimeStamp);
}
}
aOutput->Write(buf.Elements(), offset / outputChannels, &(mChunks[mChunks.Length() - 1].mTimeStamp));
if (aMixer) {
aMixer->Mix(buf.Elements(), outputChannels, GetDuration(), aOutput->GetRate());
}
aOutput->Start();
}
void
AudioNodeExternalInputStream::TrackMapEntry::ResampleInputData(AudioSegment* aSegment)
{
AudioSegment::ChunkIterator ci(*aSegment);
while (!ci.IsEnded()) {
const AudioChunk& chunk = *ci;
nsAutoTArray<const void*,2> channels;
if (chunk.GetDuration() > UINT32_MAX) {
// This will cause us to OOM or overflow below. So let's just bail.
NS_ERROR("Chunk duration out of bounds");
return;
}
uint32_t duration = uint32_t(chunk.GetDuration());
if (chunk.IsNull()) {
nsAutoTArray<AudioDataValue,1024> silence;
silence.SetLength(duration);
PodZero(silence.Elements(), silence.Length());
channels.SetLength(mResamplerChannelCount);
for (uint32_t i = 0; i < channels.Length(); ++i) {
channels[i] = silence.Elements();
}
ResampleChannels(channels, duration, AUDIO_OUTPUT_FORMAT, 0.0f);
} else if (chunk.mChannelData.Length() == mResamplerChannelCount) {
// Common case, since mResamplerChannelCount is set to the first chunk's
// number of channels.
channels.AppendElements(chunk.mChannelData);
ResampleChannels(channels, duration, chunk.mBufferFormat, chunk.mVolume);
} else {
// Uncommon case. Since downmixing requires channels to be floats,
// convert everything to floats now.
uint32_t upChannels = GetAudioChannelsSuperset(chunk.mChannelData.Length(), mResamplerChannelCount);
nsTArray<float> buffer;
if (chunk.mBufferFormat == AUDIO_FORMAT_FLOAT32) {
channels.AppendElements(chunk.mChannelData);
} else {
NS_ASSERTION(chunk.mBufferFormat == AUDIO_FORMAT_S16, "Unknown format");
if (duration > UINT32_MAX/chunk.mChannelData.Length()) {
NS_ERROR("Chunk duration out of bounds");
return;
}
buffer.SetLength(chunk.mChannelData.Length()*duration);
for (uint32_t i = 0; i < chunk.mChannelData.Length(); ++i) {
const int16_t* samples = static_cast<const int16_t*>(chunk.mChannelData[i]);
float* converted = &buffer[i*duration];
for (uint32_t j = 0; j < duration; ++j) {
converted[j] = AudioSampleToFloat(samples[j]);
}
channels.AppendElement(converted);
}
}
nsTArray<float> zeroes;
if (channels.Length() < upChannels) {
zeroes.SetLength(duration);
PodZero(zeroes.Elements(), zeroes.Length());
AudioChannelsUpMix(&channels, upChannels, zeroes.Elements());
}
if (channels.Length() == mResamplerChannelCount) {
ResampleChannels(channels, duration, AUDIO_FORMAT_FLOAT32, chunk.mVolume);
} else {
nsTArray<float> output;
if (duration > UINT32_MAX/mResamplerChannelCount) {
NS_ERROR("Chunk duration out of bounds");
return;
}
output.SetLength(duration*mResamplerChannelCount);
nsAutoTArray<float*,2> outputPtrs;
nsAutoTArray<const void*,2> outputPtrsConst;
for (uint32_t i = 0; i < mResamplerChannelCount; ++i) {
outputPtrs.AppendElement(output.Elements() + i*duration);
outputPtrsConst.AppendElement(outputPtrs[i]);
}
AudioChannelsDownMix(channels, outputPtrs.Elements(), outputPtrs.Length(), duration);
ResampleChannels(outputPtrsConst, duration, AUDIO_FORMAT_FLOAT32, chunk.mVolume);
}
}
ci.Next();
}
}
