void AudioMixerClientData::setupCodec(CodecPluginPointer codec, const QString& codecName) {
cleanupCodec(); // cleanup any previously allocated coders first
_codec = codec;
_selectedCodecName = codecName;
if (codec) {
_encoder = codec->createEncoder(AudioConstants::SAMPLE_RATE, AudioConstants::STEREO);
_decoder = codec->createDecoder(AudioConstants::SAMPLE_RATE, AudioConstants::MONO);
}
auto avatarAudioStream = getAvatarAudioStream();
if (avatarAudioStream) {
avatarAudioStream->setupCodec(codec, codecName, avatarAudioStream->isStereo() ? AudioConstants::STEREO : AudioConstants::MONO);
qCDebug(audio) << "setting AvatarAudioStream... codec:" << _selectedCodecName << "isStereo:" << avatarAudioStream->isStereo();
}
#if INJECTORS_SUPPORT_CODECS
// fixup codecs for any active injectors...
auto it = _audioStreams.begin();
while (it != _audioStreams.end()) {
SharedStreamPointer stream = it->second;
if (stream->getType() == PositionalAudioStream::Injector) {
stream->setupCodec(codec, codecName, stream->isStereo() ? AudioConstants::STEREO : AudioConstants::MONO);
}
++it;
}
#endif
}
int AudioMixerClientData::checkBuffersBeforeFrameSend() {
QWriteLocker writeLocker { &_streamsLock };
auto it = _audioStreams.begin();
while (it != _audioStreams.end()) {
SharedStreamPointer stream = it->second;
if (stream->popFrames(1, true) > 0) {
stream->updateLastPopOutputLoudnessAndTrailingLoudness();
}
static const int INJECTOR_MAX_INACTIVE_BLOCKS = 500;
// if we don't have new data for an injected stream in the last INJECTOR_MAX_INACTIVE_BLOCKS then
// we remove the injector from our streams
if (stream->getType() == PositionalAudioStream::Injector
&& stream->getConsecutiveNotMixedCount() > INJECTOR_MAX_INACTIVE_BLOCKS) {
// this is an inactive injector, pull it from our streams
// first emit that it is finished so that the HRTF objects for this source can be cleaned up
emit injectorStreamFinished(it->second->getStreamIdentifier());
// erase the stream to drop our ref to the shared pointer and remove it
it = _audioStreams.erase(it);
} else {
++it;
}
}
return (int)_audioStreams.size();
}
int AudioMixerClientData::parseData(ReceivedMessage& message) {
PacketType packetType = message.getType();
if (packetType == PacketType::AudioStreamStats) {
// skip over header, appendFlag, and num stats packed
message.seek(sizeof(quint8) + sizeof(quint16));
// read the downstream audio stream stats
message.readPrimitive(&_downstreamAudioStreamStats);
return message.getPosition();
} else {
SharedStreamPointer matchingStream;
bool isMicStream = false;
if (packetType == PacketType::MicrophoneAudioWithEcho
|| packetType == PacketType::MicrophoneAudioNoEcho
|| packetType == PacketType::SilentAudioFrame) {
QWriteLocker writeLocker { &_streamsLock };
auto micStreamIt = _audioStreams.find(QUuid());
if (micStreamIt == _audioStreams.end()) {
// we don't have a mic stream yet, so add it
// read the channel flag to see if our stream is stereo or not
message.seek(sizeof(quint16));
quint8 channelFlag;
message.readPrimitive(&channelFlag);
bool isStereo = channelFlag == 1;
auto emplaced = _audioStreams.emplace(
QUuid(),
std::unique_ptr<PositionalAudioStream> { new AvatarAudioStream(isStereo, AudioMixer::getStreamSettings()) }
);
micStreamIt = emplaced.first;
}
matchingStream = micStreamIt->second;
writeLocker.unlock();
isMicStream = true;
} else if (packetType == PacketType::InjectAudio) {
// this is injected audio
// grab the stream identifier for this injected audio
message.seek(sizeof(quint16));
QUuid streamIdentifier = QUuid::fromRfc4122(message.readWithoutCopy(NUM_BYTES_RFC4122_UUID));
bool isStereo;
message.readPrimitive(&isStereo);
QWriteLocker writeLock { &_streamsLock };
auto streamIt = _audioStreams.find(streamIdentifier);
if (streamIt == _audioStreams.end()) {
// we don't have this injected stream yet, so add it
auto emplaced = _audioStreams.emplace(
streamIdentifier,
std::unique_ptr<InjectedAudioStream> { new InjectedAudioStream(streamIdentifier, isStereo, AudioMixer::getStreamSettings()) }
);
streamIt = emplaced.first;
}
matchingStream = streamIt->second;
writeLock.unlock();
}
// seek to the beginning of the packet so that the next reader is in the right spot
message.seek(0);
// check the overflow count before we parse data
auto overflowBefore = matchingStream->getOverflowCount();
auto parseResult = matchingStream->parseData(message);
if (matchingStream->getOverflowCount() > overflowBefore) {
qDebug() << "Just overflowed on stream from" << message.getSourceID() << "at" << message.getSenderSockAddr();
qDebug() << "This stream is for" << (isMicStream ? "microphone audio" : "injected audio");
}
return parseResult;
}
return 0;
}
int AudioMixerClientData::parseData(ReceivedMessage& message) {
PacketType packetType = message.getType();
if (packetType == PacketType::AudioStreamStats) {
// skip over header, appendFlag, and num stats packed
message.seek(sizeof(quint8) + sizeof(quint16));
// read the downstream audio stream stats
message.readPrimitive(&_downstreamAudioStreamStats);
return message.getPosition();
} else {
SharedStreamPointer matchingStream;
bool isMicStream = false;
if (packetType == PacketType::MicrophoneAudioWithEcho
|| packetType == PacketType::ReplicatedMicrophoneAudioWithEcho
|| packetType == PacketType::MicrophoneAudioNoEcho
|| packetType == PacketType::ReplicatedMicrophoneAudioNoEcho
|| packetType == PacketType::SilentAudioFrame
|| packetType == PacketType::ReplicatedSilentAudioFrame) {
QWriteLocker writeLocker { &_streamsLock };
auto micStreamIt = _audioStreams.find(QUuid());
if (micStreamIt == _audioStreams.end()) {
// we don't have a mic stream yet, so add it
// hop past the sequence number that leads the packet
message.seek(sizeof(quint16));
// pull the codec string from the packet
auto codecString = message.readString();
// determine if the stream is stereo or not
bool isStereo;
if (packetType == PacketType::SilentAudioFrame
|| packetType == PacketType::ReplicatedSilentAudioFrame) {
quint16 numSilentSamples;
message.readPrimitive(&numSilentSamples);
isStereo = numSilentSamples == AudioConstants::NETWORK_FRAME_SAMPLES_STEREO;
} else {
quint8 channelFlag;
message.readPrimitive(&channelFlag);
isStereo = channelFlag == 1;
}
auto avatarAudioStream = new AvatarAudioStream(isStereo, AudioMixer::getStaticJitterFrames());
avatarAudioStream->setupCodec(_codec, _selectedCodecName, isStereo ? AudioConstants::STEREO : AudioConstants::MONO);
qCDebug(audio) << "creating new AvatarAudioStream... codec:" << _selectedCodecName << "isStereo:" << isStereo;
connect(avatarAudioStream, &InboundAudioStream::mismatchedAudioCodec,
this, &AudioMixerClientData::handleMismatchAudioFormat);
auto emplaced = _audioStreams.emplace(
QUuid(),
std::unique_ptr<PositionalAudioStream> { avatarAudioStream }
);
micStreamIt = emplaced.first;
}
matchingStream = micStreamIt->second;
writeLocker.unlock();
isMicStream = true;
} else if (packetType == PacketType::InjectAudio
|| packetType == PacketType::ReplicatedInjectAudio) {
// this is injected audio
// grab the stream identifier for this injected audio
message.seek(sizeof(quint16));
QUuid streamIdentifier = QUuid::fromRfc4122(message.readWithoutCopy(NUM_BYTES_RFC4122_UUID));
bool isStereo;
message.readPrimitive(&isStereo);
QWriteLocker writeLock { &_streamsLock };
auto streamIt = _audioStreams.find(streamIdentifier);
if (streamIt == _audioStreams.end()) {
// we don't have this injected stream yet, so add it
auto injectorStream = new InjectedAudioStream(streamIdentifier, isStereo, AudioMixer::getStaticJitterFrames());
#if INJECTORS_SUPPORT_CODECS
injectorStream->setupCodec(_codec, _selectedCodecName, isStereo ? AudioConstants::STEREO : AudioConstants::MONO);
qCDebug(audio) << "creating new injectorStream... codec:" << _selectedCodecName << "isStereo:" << isStereo;
#endif
auto emplaced = _audioStreams.emplace(
streamIdentifier,
std::unique_ptr<InjectedAudioStream> { injectorStream }
);
streamIt = emplaced.first;
}
matchingStream = streamIt->second;
writeLock.unlock();
}
// seek to the beginning of the packet so that the next reader is in the right spot
message.seek(0);
// check the overflow count before we parse data
auto overflowBefore = matchingStream->getOverflowCount();
auto parseResult = matchingStream->parseData(message);
if (matchingStream->getOverflowCount() > overflowBefore) {
qCDebug(audio) << "Just overflowed on stream from" << message.getSourceID() << "at" << message.getSenderSockAddr();
qCDebug(audio) << "This stream is for" << (isMicStream ? "microphone audio" : "injected audio");
}
return parseResult;
}
return 0;
}
void AudioMixerClientData::processStreamPacket(ReceivedMessage& message, ConcurrentAddedStreams &addedStreams) {
if (!containsValidPosition(message)) {
qDebug() << "Refusing to process audio stream from" << message.getSourceID() << "with invalid position";
return;
}
SharedStreamPointer matchingStream;
auto packetType = message.getType();
bool newStream = false;
if (packetType == PacketType::MicrophoneAudioWithEcho
|| packetType == PacketType::MicrophoneAudioNoEcho
|| packetType == PacketType::SilentAudioFrame) {
auto micStreamIt = std::find_if(_audioStreams.begin(), _audioStreams.end(), [](const SharedStreamPointer& stream){
return stream->getStreamIdentifier().isNull();
});
if (micStreamIt == _audioStreams.end()) {
// we don't have a mic stream yet, so add it
// hop past the sequence number that leads the packet
message.seek(sizeof(StreamSequenceNumber));
// pull the codec string from the packet
auto codecString = message.readString();
// determine if the stream is stereo or not
bool isStereo;
if (packetType == PacketType::SilentAudioFrame || packetType == PacketType::ReplicatedSilentAudioFrame) {
SilentSamplesBytes numSilentSamples;
message.readPrimitive(&numSilentSamples);
isStereo = numSilentSamples == AudioConstants::NETWORK_FRAME_SAMPLES_STEREO;
} else {
ChannelFlag channelFlag;
message.readPrimitive(&channelFlag);
isStereo = channelFlag == 1;
}
auto avatarAudioStream = new AvatarAudioStream(isStereo, AudioMixer::getStaticJitterFrames());
avatarAudioStream->setupCodec(_codec, _selectedCodecName, isStereo ? AudioConstants::STEREO : AudioConstants::MONO);
if (_isIgnoreRadiusEnabled) {
avatarAudioStream->enableIgnoreBox();
} else {
avatarAudioStream->disableIgnoreBox();
}
qCDebug(audio) << "creating new AvatarAudioStream... codec:" << _selectedCodecName << "isStereo:" << isStereo;
connect(avatarAudioStream, &InboundAudioStream::mismatchedAudioCodec,
this, &AudioMixerClientData::handleMismatchAudioFormat);
matchingStream = SharedStreamPointer(avatarAudioStream);
_audioStreams.push_back(matchingStream);
newStream = true;
} else {
matchingStream = *micStreamIt;
}
} else if (packetType == PacketType::InjectAudio) {
// this is injected audio
// skip the sequence number and codec string and grab the stream identifier for this injected audio
message.seek(sizeof(StreamSequenceNumber));
message.readString();
QUuid streamIdentifier = QUuid::fromRfc4122(message.readWithoutCopy(NUM_BYTES_RFC4122_UUID));
auto streamIt = std::find_if(_audioStreams.begin(), _audioStreams.end(), [&streamIdentifier](const SharedStreamPointer& stream) {
return stream->getStreamIdentifier() == streamIdentifier;
});
if (streamIt == _audioStreams.end()) {
bool isStereo;
message.readPrimitive(&isStereo);
// we don't have this injected stream yet, so add it
auto injectorStream = new InjectedAudioStream(streamIdentifier, isStereo, AudioMixer::getStaticJitterFrames());
#if INJECTORS_SUPPORT_CODECS
injectorStream->setupCodec(_codec, _selectedCodecName, isStereo ? AudioConstants::STEREO : AudioConstants::MONO);
qCDebug(audio) << "creating new injectorStream... codec:" << _selectedCodecName << "isStereo:" << isStereo;
#endif
matchingStream = SharedStreamPointer(injectorStream);
_audioStreams.push_back(matchingStream);
newStream = true;
} else {
matchingStream = *streamIt;
}
}
// seek to the beginning of the packet so that the next reader is in the right spot
message.seek(0);
// check the overflow count before we parse data
auto overflowBefore = matchingStream->getOverflowCount();
matchingStream->parseData(message);
if (matchingStream->getOverflowCount() > overflowBefore) {
qCDebug(audio) << "Just overflowed on stream" << matchingStream->getStreamIdentifier()
<< "from" << message.getSourceID();
}
if (newStream) {
// whenever a stream is added, push it to the concurrent vector of streams added this frame
addedStreams.push_back(AddedStream(getNodeID(), getNodeLocalID(), matchingStream->getStreamIdentifier(), matchingStream.get()));
}
}
