void AudioMixerClientData::setupCodec(CodecPluginPointer codec, const QString& codecName) { cleanupCodec(); // cleanup any previously allocated coders first _codec = codec; _selectedCodecName = codecName; if (codec) { _encoder = codec->createEncoder(AudioConstants::SAMPLE_RATE, AudioConstants::STEREO); _decoder = codec->createDecoder(AudioConstants::SAMPLE_RATE, AudioConstants::MONO); } auto avatarAudioStream = getAvatarAudioStream(); if (avatarAudioStream) { avatarAudioStream->setupCodec(codec, codecName, avatarAudioStream->isStereo() ? AudioConstants::STEREO : AudioConstants::MONO); qCDebug(audio) << "setting AvatarAudioStream... codec:" << _selectedCodecName << "isStereo:" << avatarAudioStream->isStereo(); } #if INJECTORS_SUPPORT_CODECS // fixup codecs for any active injectors... auto it = _audioStreams.begin(); while (it != _audioStreams.end()) { SharedStreamPointer stream = it->second; if (stream->getType() == PositionalAudioStream::Injector) { stream->setupCodec(codec, codecName, stream->isStereo() ? AudioConstants::STEREO : AudioConstants::MONO); } ++it; } #endif }
void AudioMixerClientData::negotiateAudioFormat(ReceivedMessage& message, const SharedNodePointer& node) { quint8 numberOfCodecs; message.readPrimitive(&numberOfCodecs); std::vector<QString> codecs; for (auto i = 0; i < numberOfCodecs; i++) { codecs.push_back(message.readString()); } const std::pair<QString, CodecPluginPointer> codec = AudioMixer::negotiateCodec(codecs); setupCodec(codec.second, codec.first); sendSelectAudioFormat(node, codec.first); }
void SWDecoder::operator()() { int res = 0; std::cout << "SWDecoder thread started" << std::endl; setupBuffer(); setupCodec(); while(!mFinish) { decodeStream(); boost::thread::sleep(boost::get_system_time() + boost::posix_time::milliseconds(5)); } releaseCodec(); std::cout << "SWDecoder thread stopped" << std::endl; }
void AudioMixerClientData::setupCodecForReplicatedAgent(QSharedPointer<ReceivedMessage> message) { // hop past the sequence number that leads the packet message->seek(sizeof(quint16)); // pull the codec string from the packet auto codecString = message->readString(); if (codecString != _selectedCodecName) { qCDebug(audio) << "Manually setting codec for replicated agent" << uuidStringWithoutCurlyBraces(getNodeID()) << "-" << codecString; const std::pair<QString, CodecPluginPointer> codec = AudioMixer::negotiateCodec({ codecString }); setupCodec(codec.second, codec.first); // seek back to the beginning of the message so other readers are in the right place message->seek(0); } }
int AudioMixerClientData::parseData(ReceivedMessage& message) { PacketType packetType = message.getType(); if (packetType == PacketType::AudioStreamStats) { // skip over header, appendFlag, and num stats packed message.seek(sizeof(quint8) + sizeof(quint16)); // read the downstream audio stream stats message.readPrimitive(&_downstreamAudioStreamStats); return message.getPosition(); } else { SharedStreamPointer matchingStream; bool isMicStream = false; if (packetType == PacketType::MicrophoneAudioWithEcho || packetType == PacketType::ReplicatedMicrophoneAudioWithEcho || packetType == PacketType::MicrophoneAudioNoEcho || packetType == PacketType::ReplicatedMicrophoneAudioNoEcho || packetType == PacketType::SilentAudioFrame || packetType == PacketType::ReplicatedSilentAudioFrame) { QWriteLocker writeLocker { &_streamsLock }; auto micStreamIt = _audioStreams.find(QUuid()); if (micStreamIt == _audioStreams.end()) { // we don't have a mic stream yet, so add it // hop past the sequence number that leads the packet message.seek(sizeof(quint16)); // pull the codec string from the packet auto codecString = message.readString(); // determine if the stream is stereo or not bool isStereo; if (packetType == PacketType::SilentAudioFrame || packetType == PacketType::ReplicatedSilentAudioFrame) { quint16 numSilentSamples; message.readPrimitive(&numSilentSamples); isStereo = numSilentSamples == AudioConstants::NETWORK_FRAME_SAMPLES_STEREO; } else { quint8 channelFlag; message.readPrimitive(&channelFlag); isStereo = channelFlag == 1; } auto avatarAudioStream = new AvatarAudioStream(isStereo, AudioMixer::getStaticJitterFrames()); avatarAudioStream->setupCodec(_codec, _selectedCodecName, isStereo ? AudioConstants::STEREO : AudioConstants::MONO); qCDebug(audio) << "creating new AvatarAudioStream... codec:" << _selectedCodecName << "isStereo:" << isStereo; connect(avatarAudioStream, &InboundAudioStream::mismatchedAudioCodec, this, &AudioMixerClientData::handleMismatchAudioFormat); auto emplaced = _audioStreams.emplace( QUuid(), std::unique_ptr<PositionalAudioStream> { avatarAudioStream } ); micStreamIt = emplaced.first; } matchingStream = micStreamIt->second; writeLocker.unlock(); isMicStream = true; } else if (packetType == PacketType::InjectAudio || packetType == PacketType::ReplicatedInjectAudio) { // this is injected audio // grab the stream identifier for this injected audio message.seek(sizeof(quint16)); QUuid streamIdentifier = QUuid::fromRfc4122(message.readWithoutCopy(NUM_BYTES_RFC4122_UUID)); bool isStereo; message.readPrimitive(&isStereo); QWriteLocker writeLock { &_streamsLock }; auto streamIt = _audioStreams.find(streamIdentifier); if (streamIt == _audioStreams.end()) { // we don't have this injected stream yet, so add it auto injectorStream = new InjectedAudioStream(streamIdentifier, isStereo, AudioMixer::getStaticJitterFrames()); #if INJECTORS_SUPPORT_CODECS injectorStream->setupCodec(_codec, _selectedCodecName, isStereo ? AudioConstants::STEREO : AudioConstants::MONO); qCDebug(audio) << "creating new injectorStream... codec:" << _selectedCodecName << "isStereo:" << isStereo; #endif auto emplaced = _audioStreams.emplace( streamIdentifier, std::unique_ptr<InjectedAudioStream> { injectorStream } ); streamIt = emplaced.first; } matchingStream = streamIt->second; writeLock.unlock(); } // seek to the beginning of the packet so that the next reader is in the right spot message.seek(0); // check the overflow count before we parse data auto overflowBefore = matchingStream->getOverflowCount(); auto parseResult = matchingStream->parseData(message); if (matchingStream->getOverflowCount() > overflowBefore) { qCDebug(audio) << "Just overflowed on stream from" << message.getSourceID() << "at" << message.getSenderSockAddr(); qCDebug(audio) << "This stream is for" << (isMicStream ? "microphone audio" : "injected audio"); } return parseResult; } return 0; }
void AudioMixerClientData::processStreamPacket(ReceivedMessage& message, ConcurrentAddedStreams &addedStreams) { if (!containsValidPosition(message)) { qDebug() << "Refusing to process audio stream from" << message.getSourceID() << "with invalid position"; return; } SharedStreamPointer matchingStream; auto packetType = message.getType(); bool newStream = false; if (packetType == PacketType::MicrophoneAudioWithEcho || packetType == PacketType::MicrophoneAudioNoEcho || packetType == PacketType::SilentAudioFrame) { auto micStreamIt = std::find_if(_audioStreams.begin(), _audioStreams.end(), [](const SharedStreamPointer& stream){ return stream->getStreamIdentifier().isNull(); }); if (micStreamIt == _audioStreams.end()) { // we don't have a mic stream yet, so add it // hop past the sequence number that leads the packet message.seek(sizeof(StreamSequenceNumber)); // pull the codec string from the packet auto codecString = message.readString(); // determine if the stream is stereo or not bool isStereo; if (packetType == PacketType::SilentAudioFrame || packetType == PacketType::ReplicatedSilentAudioFrame) { SilentSamplesBytes numSilentSamples; message.readPrimitive(&numSilentSamples); isStereo = numSilentSamples == AudioConstants::NETWORK_FRAME_SAMPLES_STEREO; } else { ChannelFlag channelFlag; message.readPrimitive(&channelFlag); isStereo = channelFlag == 1; } auto avatarAudioStream = new AvatarAudioStream(isStereo, AudioMixer::getStaticJitterFrames()); avatarAudioStream->setupCodec(_codec, _selectedCodecName, isStereo ? AudioConstants::STEREO : AudioConstants::MONO); if (_isIgnoreRadiusEnabled) { avatarAudioStream->enableIgnoreBox(); } else { avatarAudioStream->disableIgnoreBox(); } qCDebug(audio) << "creating new AvatarAudioStream... codec:" << _selectedCodecName << "isStereo:" << isStereo; connect(avatarAudioStream, &InboundAudioStream::mismatchedAudioCodec, this, &AudioMixerClientData::handleMismatchAudioFormat); matchingStream = SharedStreamPointer(avatarAudioStream); _audioStreams.push_back(matchingStream); newStream = true; } else { matchingStream = *micStreamIt; } } else if (packetType == PacketType::InjectAudio) { // this is injected audio // skip the sequence number and codec string and grab the stream identifier for this injected audio message.seek(sizeof(StreamSequenceNumber)); message.readString(); QUuid streamIdentifier = QUuid::fromRfc4122(message.readWithoutCopy(NUM_BYTES_RFC4122_UUID)); auto streamIt = std::find_if(_audioStreams.begin(), _audioStreams.end(), [&streamIdentifier](const SharedStreamPointer& stream) { return stream->getStreamIdentifier() == streamIdentifier; }); if (streamIt == _audioStreams.end()) { bool isStereo; message.readPrimitive(&isStereo); // we don't have this injected stream yet, so add it auto injectorStream = new InjectedAudioStream(streamIdentifier, isStereo, AudioMixer::getStaticJitterFrames()); #if INJECTORS_SUPPORT_CODECS injectorStream->setupCodec(_codec, _selectedCodecName, isStereo ? AudioConstants::STEREO : AudioConstants::MONO); qCDebug(audio) << "creating new injectorStream... codec:" << _selectedCodecName << "isStereo:" << isStereo; #endif matchingStream = SharedStreamPointer(injectorStream); _audioStreams.push_back(matchingStream); newStream = true; } else { matchingStream = *streamIt; } } // seek to the beginning of the packet so that the next reader is in the right spot message.seek(0); // check the overflow count before we parse data auto overflowBefore = matchingStream->getOverflowCount(); matchingStream->parseData(message); if (matchingStream->getOverflowCount() > overflowBefore) { qCDebug(audio) << "Just overflowed on stream" << matchingStream->getStreamIdentifier() << "from" << message.getSourceID(); } if (newStream) { // whenever a stream is added, push it to the concurrent vector of streams added this frame addedStreams.push_back(AddedStream(getNodeID(), getNodeLocalID(), matchingStream->getStreamIdentifier(), matchingStream.get())); } }
void AudioMixer::handleNegotiateAudioFormat(QSharedPointer<ReceivedMessage> message, SharedNodePointer sendingNode) { QStringList availableCodecs; auto codecPlugins = PluginManager::getInstance()->getCodecPlugins(); if (codecPlugins.size() > 0) { for (auto& plugin : codecPlugins) { auto codecName = plugin->getName(); qDebug() << "Codec available:" << codecName; availableCodecs.append(codecName); } } else { qDebug() << "No Codecs available..."; } CodecPluginPointer selectedCodec; QString selectedCodecName; QStringList codecPreferenceList = _codecPreferenceOrder.split(","); // read the codecs requested by the client const int MAX_PREFERENCE = 99999; int preferredCodecIndex = MAX_PREFERENCE; QString preferredCodec; quint8 numberOfCodecs = 0; message->readPrimitive(&numberOfCodecs); qDebug() << "numberOfCodecs:" << numberOfCodecs; QStringList codecList; for (quint16 i = 0; i < numberOfCodecs; i++) { QString requestedCodec = message->readString(); int preferenceOfThisCodec = codecPreferenceList.indexOf(requestedCodec); bool codecAvailable = availableCodecs.contains(requestedCodec); qDebug() << "requestedCodec:" << requestedCodec << "preference:" << preferenceOfThisCodec << "available:" << codecAvailable; if (codecAvailable) { codecList.append(requestedCodec); if (preferenceOfThisCodec >= 0 && preferenceOfThisCodec < preferredCodecIndex) { qDebug() << "This codec is preferred..."; selectedCodecName = requestedCodec; preferredCodecIndex = preferenceOfThisCodec; } } } qDebug() << "all requested and available codecs:" << codecList; // choose first codec if (!selectedCodecName.isEmpty()) { if (codecPlugins.size() > 0) { for (auto& plugin : codecPlugins) { if (selectedCodecName == plugin->getName()) { qDebug() << "Selecting codec:" << selectedCodecName; selectedCodec = plugin; break; } } } } auto clientData = dynamic_cast<AudioMixerClientData*>(sendingNode->getLinkedData()); // FIXME - why would we not have client data at this point?? if (!clientData) { qDebug() << "UNEXPECTED -- didn't have node linked data in " << __FUNCTION__; sendingNode->setLinkedData(std::unique_ptr<NodeData> { new AudioMixerClientData(sendingNode->getUUID()) }); clientData = dynamic_cast<AudioMixerClientData*>(sendingNode->getLinkedData()); connect(clientData, &AudioMixerClientData::injectorStreamFinished, this, &AudioMixer::removeHRTFsForFinishedInjector); } clientData->setupCodec(selectedCodec, selectedCodecName); qDebug() << "selectedCodecName:" << selectedCodecName; clientData->sendSelectAudioFormat(sendingNode, selectedCodecName); }