void AudioMixerClientData::setupCodec(CodecPluginPointer codec, const QString& codecName) {
cleanupCodec(); // cleanup any previously allocated coders first
_codec = codec;
_selectedCodecName = codecName;
if (codec) {
_encoder = codec->createEncoder(AudioConstants::SAMPLE_RATE, AudioConstants::STEREO);
_decoder = codec->createDecoder(AudioConstants::SAMPLE_RATE, AudioConstants::MONO);
}
auto avatarAudioStream = getAvatarAudioStream();
if (avatarAudioStream) {
avatarAudioStream->setupCodec(codec, codecName, avatarAudioStream->isStereo() ? AudioConstants::STEREO : AudioConstants::MONO);
qCDebug(audio) << "setting AvatarAudioStream... codec:" << _selectedCodecName << "isStereo:" << avatarAudioStream->isStereo();
}
#if INJECTORS_SUPPORT_CODECS
// fixup codecs for any active injectors...
auto it = _audioStreams.begin();
while (it != _audioStreams.end()) {
SharedStreamPointer stream = it->second;
if (stream->getType() == PositionalAudioStream::Injector) {
stream->setupCodec(codec, codecName, stream->isStereo() ? AudioConstants::STEREO : AudioConstants::MONO);
}
++it;
}
#endif
}
void AudioMixerClientData::negotiateAudioFormat(ReceivedMessage& message, const SharedNodePointer& node) {
quint8 numberOfCodecs;
message.readPrimitive(&numberOfCodecs);
std::vector<QString> codecs;
for (auto i = 0; i < numberOfCodecs; i++) {
codecs.push_back(message.readString());
}
const std::pair<QString, CodecPluginPointer> codec = AudioMixer::negotiateCodec(codecs);
setupCodec(codec.second, codec.first);
sendSelectAudioFormat(node, codec.first);
}
void SWDecoder::operator()()
{
int res = 0;
std::cout << "SWDecoder thread started" << std::endl;
setupBuffer();
setupCodec();
while(!mFinish)
{
decodeStream();
boost::thread::sleep(boost::get_system_time() + boost::posix_time::milliseconds(5));
}
releaseCodec();
std::cout << "SWDecoder thread stopped" << std::endl;
}
void AudioMixerClientData::setupCodecForReplicatedAgent(QSharedPointer<ReceivedMessage> message) {
// hop past the sequence number that leads the packet
message->seek(sizeof(quint16));
// pull the codec string from the packet
auto codecString = message->readString();
if (codecString != _selectedCodecName) {
qCDebug(audio) << "Manually setting codec for replicated agent" << uuidStringWithoutCurlyBraces(getNodeID())
<< "-" << codecString;
const std::pair<QString, CodecPluginPointer> codec = AudioMixer::negotiateCodec({ codecString });
setupCodec(codec.second, codec.first);
// seek back to the beginning of the message so other readers are in the right place
message->seek(0);
}
}
int AudioMixerClientData::parseData(ReceivedMessage& message) {
PacketType packetType = message.getType();
if (packetType == PacketType::AudioStreamStats) {
// skip over header, appendFlag, and num stats packed
message.seek(sizeof(quint8) + sizeof(quint16));
// read the downstream audio stream stats
message.readPrimitive(&_downstreamAudioStreamStats);
return message.getPosition();
} else {
SharedStreamPointer matchingStream;
bool isMicStream = false;
if (packetType == PacketType::MicrophoneAudioWithEcho
|| packetType == PacketType::ReplicatedMicrophoneAudioWithEcho
|| packetType == PacketType::MicrophoneAudioNoEcho
|| packetType == PacketType::ReplicatedMicrophoneAudioNoEcho
|| packetType == PacketType::SilentAudioFrame
|| packetType == PacketType::ReplicatedSilentAudioFrame) {
QWriteLocker writeLocker { &_streamsLock };
auto micStreamIt = _audioStreams.find(QUuid());
if (micStreamIt == _audioStreams.end()) {
// we don't have a mic stream yet, so add it
// hop past the sequence number that leads the packet
message.seek(sizeof(quint16));
// pull the codec string from the packet
auto codecString = message.readString();
// determine if the stream is stereo or not
bool isStereo;
if (packetType == PacketType::SilentAudioFrame
|| packetType == PacketType::ReplicatedSilentAudioFrame) {
quint16 numSilentSamples;
message.readPrimitive(&numSilentSamples);
isStereo = numSilentSamples == AudioConstants::NETWORK_FRAME_SAMPLES_STEREO;
} else {
quint8 channelFlag;
message.readPrimitive(&channelFlag);
isStereo = channelFlag == 1;
}
auto avatarAudioStream = new AvatarAudioStream(isStereo, AudioMixer::getStaticJitterFrames());
avatarAudioStream->setupCodec(_codec, _selectedCodecName, isStereo ? AudioConstants::STEREO : AudioConstants::MONO);
qCDebug(audio) << "creating new AvatarAudioStream... codec:" << _selectedCodecName << "isStereo:" << isStereo;
connect(avatarAudioStream, &InboundAudioStream::mismatchedAudioCodec,
this, &AudioMixerClientData::handleMismatchAudioFormat);
auto emplaced = _audioStreams.emplace(
QUuid(),
std::unique_ptr<PositionalAudioStream> { avatarAudioStream }
);
micStreamIt = emplaced.first;
}
matchingStream = micStreamIt->second;
writeLocker.unlock();
isMicStream = true;
} else if (packetType == PacketType::InjectAudio
|| packetType == PacketType::ReplicatedInjectAudio) {
// this is injected audio
// grab the stream identifier for this injected audio
message.seek(sizeof(quint16));
QUuid streamIdentifier = QUuid::fromRfc4122(message.readWithoutCopy(NUM_BYTES_RFC4122_UUID));
bool isStereo;
message.readPrimitive(&isStereo);
QWriteLocker writeLock { &_streamsLock };
auto streamIt = _audioStreams.find(streamIdentifier);
if (streamIt == _audioStreams.end()) {
// we don't have this injected stream yet, so add it
auto injectorStream = new InjectedAudioStream(streamIdentifier, isStereo, AudioMixer::getStaticJitterFrames());
#if INJECTORS_SUPPORT_CODECS
injectorStream->setupCodec(_codec, _selectedCodecName, isStereo ? AudioConstants::STEREO : AudioConstants::MONO);
qCDebug(audio) << "creating new injectorStream... codec:" << _selectedCodecName << "isStereo:" << isStereo;
#endif
auto emplaced = _audioStreams.emplace(
streamIdentifier,
std::unique_ptr<InjectedAudioStream> { injectorStream }
);
streamIt = emplaced.first;
}
matchingStream = streamIt->second;
writeLock.unlock();
}
// seek to the beginning of the packet so that the next reader is in the right spot
message.seek(0);
// check the overflow count before we parse data
auto overflowBefore = matchingStream->getOverflowCount();
auto parseResult = matchingStream->parseData(message);
if (matchingStream->getOverflowCount() > overflowBefore) {
qCDebug(audio) << "Just overflowed on stream from" << message.getSourceID() << "at" << message.getSenderSockAddr();
qCDebug(audio) << "This stream is for" << (isMicStream ? "microphone audio" : "injected audio");
}
return parseResult;
}
return 0;
}
void AudioMixerClientData::processStreamPacket(ReceivedMessage& message, ConcurrentAddedStreams &addedStreams) {
if (!containsValidPosition(message)) {
qDebug() << "Refusing to process audio stream from" << message.getSourceID() << "with invalid position";
return;
}
SharedStreamPointer matchingStream;
auto packetType = message.getType();
bool newStream = false;
if (packetType == PacketType::MicrophoneAudioWithEcho
|| packetType == PacketType::MicrophoneAudioNoEcho
|| packetType == PacketType::SilentAudioFrame) {
auto micStreamIt = std::find_if(_audioStreams.begin(), _audioStreams.end(), [](const SharedStreamPointer& stream){
return stream->getStreamIdentifier().isNull();
});
if (micStreamIt == _audioStreams.end()) {
// we don't have a mic stream yet, so add it
// hop past the sequence number that leads the packet
message.seek(sizeof(StreamSequenceNumber));
// pull the codec string from the packet
auto codecString = message.readString();
// determine if the stream is stereo or not
bool isStereo;
if (packetType == PacketType::SilentAudioFrame || packetType == PacketType::ReplicatedSilentAudioFrame) {
SilentSamplesBytes numSilentSamples;
message.readPrimitive(&numSilentSamples);
isStereo = numSilentSamples == AudioConstants::NETWORK_FRAME_SAMPLES_STEREO;
} else {
ChannelFlag channelFlag;
message.readPrimitive(&channelFlag);
isStereo = channelFlag == 1;
}
auto avatarAudioStream = new AvatarAudioStream(isStereo, AudioMixer::getStaticJitterFrames());
avatarAudioStream->setupCodec(_codec, _selectedCodecName, isStereo ? AudioConstants::STEREO : AudioConstants::MONO);
if (_isIgnoreRadiusEnabled) {
avatarAudioStream->enableIgnoreBox();
} else {
avatarAudioStream->disableIgnoreBox();
}
qCDebug(audio) << "creating new AvatarAudioStream... codec:" << _selectedCodecName << "isStereo:" << isStereo;
connect(avatarAudioStream, &InboundAudioStream::mismatchedAudioCodec,
this, &AudioMixerClientData::handleMismatchAudioFormat);
matchingStream = SharedStreamPointer(avatarAudioStream);
_audioStreams.push_back(matchingStream);
newStream = true;
} else {
matchingStream = *micStreamIt;
}
} else if (packetType == PacketType::InjectAudio) {
// this is injected audio
// skip the sequence number and codec string and grab the stream identifier for this injected audio
message.seek(sizeof(StreamSequenceNumber));
message.readString();
QUuid streamIdentifier = QUuid::fromRfc4122(message.readWithoutCopy(NUM_BYTES_RFC4122_UUID));
auto streamIt = std::find_if(_audioStreams.begin(), _audioStreams.end(), [&streamIdentifier](const SharedStreamPointer& stream) {
return stream->getStreamIdentifier() == streamIdentifier;
});
if (streamIt == _audioStreams.end()) {
bool isStereo;
message.readPrimitive(&isStereo);
// we don't have this injected stream yet, so add it
auto injectorStream = new InjectedAudioStream(streamIdentifier, isStereo, AudioMixer::getStaticJitterFrames());
#if INJECTORS_SUPPORT_CODECS
injectorStream->setupCodec(_codec, _selectedCodecName, isStereo ? AudioConstants::STEREO : AudioConstants::MONO);
qCDebug(audio) << "creating new injectorStream... codec:" << _selectedCodecName << "isStereo:" << isStereo;
#endif
matchingStream = SharedStreamPointer(injectorStream);
_audioStreams.push_back(matchingStream);
newStream = true;
} else {
matchingStream = *streamIt;
}
}
// seek to the beginning of the packet so that the next reader is in the right spot
message.seek(0);
// check the overflow count before we parse data
auto overflowBefore = matchingStream->getOverflowCount();
matchingStream->parseData(message);
if (matchingStream->getOverflowCount() > overflowBefore) {
qCDebug(audio) << "Just overflowed on stream" << matchingStream->getStreamIdentifier()
<< "from" << message.getSourceID();
}
if (newStream) {
// whenever a stream is added, push it to the concurrent vector of streams added this frame
addedStreams.push_back(AddedStream(getNodeID(), getNodeLocalID(), matchingStream->getStreamIdentifier(), matchingStream.get()));
}
}
void AudioMixer::handleNegotiateAudioFormat(QSharedPointer<ReceivedMessage> message, SharedNodePointer sendingNode) {
QStringList availableCodecs;
auto codecPlugins = PluginManager::getInstance()->getCodecPlugins();
if (codecPlugins.size() > 0) {
for (auto& plugin : codecPlugins) {
auto codecName = plugin->getName();
qDebug() << "Codec available:" << codecName;
availableCodecs.append(codecName);
}
} else {
qDebug() << "No Codecs available...";
}
CodecPluginPointer selectedCodec;
QString selectedCodecName;
QStringList codecPreferenceList = _codecPreferenceOrder.split(",");
// read the codecs requested by the client
const int MAX_PREFERENCE = 99999;
int preferredCodecIndex = MAX_PREFERENCE;
QString preferredCodec;
quint8 numberOfCodecs = 0;
message->readPrimitive(&numberOfCodecs);
qDebug() << "numberOfCodecs:" << numberOfCodecs;
QStringList codecList;
for (quint16 i = 0; i < numberOfCodecs; i++) {
QString requestedCodec = message->readString();
int preferenceOfThisCodec = codecPreferenceList.indexOf(requestedCodec);
bool codecAvailable = availableCodecs.contains(requestedCodec);
qDebug() << "requestedCodec:" << requestedCodec << "preference:" << preferenceOfThisCodec << "available:" << codecAvailable;
if (codecAvailable) {
codecList.append(requestedCodec);
if (preferenceOfThisCodec >= 0 && preferenceOfThisCodec < preferredCodecIndex) {
qDebug() << "This codec is preferred...";
selectedCodecName = requestedCodec;
preferredCodecIndex = preferenceOfThisCodec;
}
}
}
qDebug() << "all requested and available codecs:" << codecList;
// choose first codec
if (!selectedCodecName.isEmpty()) {
if (codecPlugins.size() > 0) {
for (auto& plugin : codecPlugins) {
if (selectedCodecName == plugin->getName()) {
qDebug() << "Selecting codec:" << selectedCodecName;
selectedCodec = plugin;
break;
}
}
}
}
auto clientData = dynamic_cast<AudioMixerClientData*>(sendingNode->getLinkedData());
// FIXME - why would we not have client data at this point??
if (!clientData) {
qDebug() << "UNEXPECTED -- didn't have node linked data in " << __FUNCTION__;
sendingNode->setLinkedData(std::unique_ptr<NodeData> { new AudioMixerClientData(sendingNode->getUUID()) });
clientData = dynamic_cast<AudioMixerClientData*>(sendingNode->getLinkedData());
connect(clientData, &AudioMixerClientData::injectorStreamFinished, this, &AudioMixer::removeHRTFsForFinishedInjector);
}
clientData->setupCodec(selectedCodec, selectedCodecName);
qDebug() << "selectedCodecName:" << selectedCodecName;
clientData->sendSelectAudioFormat(sendingNode, selectedCodecName);
}
