void sendEnvironmentPacket(const SharedNodePointer& node, AudioMixerClientData& data) { bool hasReverb = false; float reverbTime, wetLevel; auto& reverbSettings = AudioMixer::getReverbSettings(); auto& audioZones = AudioMixer::getAudioZones(); AvatarAudioStream* stream = data.getAvatarAudioStream(); glm::vec3 streamPosition = stream->getPosition(); // find reverb properties for (int i = 0; i < reverbSettings.size(); ++i) { AABox box = audioZones[reverbSettings[i].zone]; if (box.contains(streamPosition)) { hasReverb = true; reverbTime = reverbSettings[i].reverbTime; wetLevel = reverbSettings[i].wetLevel; break; } } // check if data changed bool dataChanged = (stream->hasReverb() != hasReverb) || (stream->hasReverb() && (stream->getRevebTime() != reverbTime || stream->getWetLevel() != wetLevel)); if (dataChanged) { // update stream if (hasReverb) { stream->setReverb(reverbTime, wetLevel); } else { stream->clearReverb(); } } // send packet at change or every so often float CHANCE_OF_SEND = 0.01f; bool sendData = dataChanged || (randFloat() < CHANCE_OF_SEND); if (sendData) { // size the packet unsigned char bitset = 0; int packetSize = sizeof(bitset); if (hasReverb) { packetSize += sizeof(reverbTime) + sizeof(wetLevel); } // write the packet auto envPacket = NLPacket::create(PacketType::AudioEnvironment, packetSize); if (hasReverb) { setAtBit(bitset, HAS_REVERB_BIT); } envPacket->writePrimitive(bitset); if (hasReverb) { envPacket->writePrimitive(reverbTime); envPacket->writePrimitive(wetLevel); } // send the packet DependencyManager::get<NodeList>()->sendPacket(std::move(envPacket), *node); } }
void AudioMixerSlave::mix(const SharedNodePointer& node) { // check that the node is valid AudioMixerClientData* data = (AudioMixerClientData*)node->getLinkedData(); if (data == nullptr) { return; } if (node->isUpstream()) { return; } // check that the stream is valid auto avatarStream = data->getAvatarAudioStream(); if (avatarStream == nullptr) { return; } // send mute packet, if necessary if (AudioMixer::shouldMute(avatarStream->getQuietestFrameLoudness()) || data->shouldMuteClient()) { sendMutePacket(node, *data); } // send audio packets, if necessary if (node->getType() == NodeType::Agent && node->getActiveSocket()) { ++stats.sumListeners; // mix the audio bool mixHasAudio = prepareMix(node); // send audio packet if (mixHasAudio || data->shouldFlushEncoder()) { QByteArray encodedBuffer; if (mixHasAudio) { // encode the audio QByteArray decodedBuffer(reinterpret_cast<char*>(_bufferSamples), AudioConstants::NETWORK_FRAME_BYTES_STEREO); data->encode(decodedBuffer, encodedBuffer); } else { // time to flush (resets shouldFlush until the next encode) data->encodeFrameOfZeros(encodedBuffer); } sendMixPacket(node, *data, encodedBuffer); } else { ++stats.sumListenersSilent; sendSilentPacket(node, *data); } // send environment packet sendEnvironmentPacket(node, *data); // send stats packet (about every second) const unsigned int NUM_FRAMES_PER_SEC = (int)ceil(AudioConstants::NETWORK_FRAMES_PER_SEC); if (data->shouldSendStats(_frame % NUM_FRAMES_PER_SEC)) { data->sendAudioStreamStatsPackets(node); } } }
void AudioMixer::sendAudioEnvironmentPacket(SharedNodePointer node) { // Send stream properties bool hasReverb = false; float reverbTime, wetLevel; // find reverb properties for (int i = 0; i < _zoneReverbSettings.size(); ++i) { AudioMixerClientData* data = static_cast<AudioMixerClientData*>(node->getLinkedData()); glm::vec3 streamPosition = data->getAvatarAudioStream()->getPosition(); AABox box = _audioZones[_zoneReverbSettings[i].zone]; if (box.contains(streamPosition)) { hasReverb = true; reverbTime = _zoneReverbSettings[i].reverbTime; wetLevel = _zoneReverbSettings[i].wetLevel; // Modulate wet level with distance to wall float MIN_ATTENUATION_DISTANCE = 2.0f; float MAX_ATTENUATION = -12; // dB glm::vec3 distanceToWalls = (box.getDimensions() / 2.0f) - glm::abs(streamPosition - box.calcCenter()); float distanceToClosestWall = glm::min(distanceToWalls.x, distanceToWalls.z); if (distanceToClosestWall < MIN_ATTENUATION_DISTANCE) { wetLevel += MAX_ATTENUATION * (1.0f - distanceToClosestWall / MIN_ATTENUATION_DISTANCE); } break; } } AudioMixerClientData* nodeData = static_cast<AudioMixerClientData*>(node->getLinkedData()); AvatarAudioStream* stream = nodeData->getAvatarAudioStream(); bool dataChanged = (stream->hasReverb() != hasReverb) || (stream->hasReverb() && (stream->getRevebTime() != reverbTime || stream->getWetLevel() != wetLevel)); if (dataChanged) { // Update stream if (hasReverb) { stream->setReverb(reverbTime, wetLevel); } else { stream->clearReverb(); } } // Send at change or every so often float CHANCE_OF_SEND = 0.01f; bool sendData = dataChanged || (randFloat() < CHANCE_OF_SEND); if (sendData) { auto nodeList = DependencyManager::get<NodeList>(); unsigned char bitset = 0; int packetSize = sizeof(bitset); if (hasReverb) { packetSize += sizeof(reverbTime) + sizeof(wetLevel); } auto envPacket = NLPacket::create(PacketType::AudioEnvironment, packetSize); if (hasReverb) { setAtBit(bitset, HAS_REVERB_BIT); } envPacket->writePrimitive(bitset); if (hasReverb) { envPacket->writePrimitive(reverbTime); envPacket->writePrimitive(wetLevel); } nodeList->sendPacket(std::move(envPacket), *node); } }
void AudioMixer::run() { ThreadedAssignment::commonInit(AUDIO_MIXER_LOGGING_TARGET_NAME, NodeType::AudioMixer); auto nodeList = DependencyManager::get<NodeList>(); nodeList->addNodeTypeToInterestSet(NodeType::Agent); nodeList->linkedDataCreateCallback = [](Node* node) { node->setLinkedData(new AudioMixerClientData()); }; // wait until we have the domain-server settings, otherwise we bail DomainHandler& domainHandler = nodeList->getDomainHandler(); qDebug() << "Waiting for domain settings from domain-server."; // block until we get the settingsRequestComplete signal QEventLoop loop; connect(&domainHandler, &DomainHandler::settingsReceived, &loop, &QEventLoop::quit); connect(&domainHandler, &DomainHandler::settingsReceiveFail, &loop, &QEventLoop::quit); domainHandler.requestDomainSettings(); loop.exec(); if (domainHandler.getSettingsObject().isEmpty()) { qDebug() << "Failed to retreive settings object from domain-server. Bailing on assignment."; setFinished(true); return; } const QJsonObject& settingsObject = domainHandler.getSettingsObject(); // check the settings object to see if we have anything we can parse out parseSettingsObject(settingsObject); int nextFrame = 0; QElapsedTimer timer; timer.start(); int usecToSleep = AudioConstants::NETWORK_FRAME_USECS; const int TRAILING_AVERAGE_FRAMES = 100; int framesSinceCutoffEvent = TRAILING_AVERAGE_FRAMES; while (!_isFinished) { const float STRUGGLE_TRIGGER_SLEEP_PERCENTAGE_THRESHOLD = 0.10f; const float BACK_OFF_TRIGGER_SLEEP_PERCENTAGE_THRESHOLD = 0.20f; const float RATIO_BACK_OFF = 0.02f; const float CURRENT_FRAME_RATIO = 1.0f / TRAILING_AVERAGE_FRAMES; const float PREVIOUS_FRAMES_RATIO = 1.0f - CURRENT_FRAME_RATIO; if (usecToSleep < 0) { usecToSleep = 0; } _trailingSleepRatio = (PREVIOUS_FRAMES_RATIO * _trailingSleepRatio) + (usecToSleep * CURRENT_FRAME_RATIO / (float) AudioConstants::NETWORK_FRAME_USECS); float lastCutoffRatio = _performanceThrottlingRatio; bool hasRatioChanged = false; if (framesSinceCutoffEvent >= TRAILING_AVERAGE_FRAMES) { if (_trailingSleepRatio <= STRUGGLE_TRIGGER_SLEEP_PERCENTAGE_THRESHOLD) { // we're struggling - change our min required loudness to reduce some load _performanceThrottlingRatio = _performanceThrottlingRatio + (0.5f * (1.0f - _performanceThrottlingRatio)); qDebug() << "Mixer is struggling, sleeping" << _trailingSleepRatio * 100 << "% of frame time. Old cutoff was" << lastCutoffRatio << "and is now" << _performanceThrottlingRatio; hasRatioChanged = true; } else if (_trailingSleepRatio >= BACK_OFF_TRIGGER_SLEEP_PERCENTAGE_THRESHOLD && _performanceThrottlingRatio != 0) { // we've recovered and can back off the required loudness _performanceThrottlingRatio = _performanceThrottlingRatio - RATIO_BACK_OFF; if (_performanceThrottlingRatio < 0) { _performanceThrottlingRatio = 0; } qDebug() << "Mixer is recovering, sleeping" << _trailingSleepRatio * 100 << "% of frame time. Old cutoff was" << lastCutoffRatio << "and is now" << _performanceThrottlingRatio; hasRatioChanged = true; } if (hasRatioChanged) { // set out min audability threshold from the new ratio _minAudibilityThreshold = LOUDNESS_TO_DISTANCE_RATIO / (2.0f * (1.0f - _performanceThrottlingRatio)); qDebug() << "Minimum audability required to be mixed is now" << _minAudibilityThreshold; framesSinceCutoffEvent = 0; } } if (!hasRatioChanged) { ++framesSinceCutoffEvent; } quint64 now = usecTimestampNow(); if (now - _lastPerSecondCallbackTime > USECS_PER_SECOND) { perSecondActions(); _lastPerSecondCallbackTime = now; } nodeList->eachNode([&](const SharedNodePointer& node) { if (node->getLinkedData()) { AudioMixerClientData* nodeData = (AudioMixerClientData*)node->getLinkedData(); // this function will attempt to pop a frame from each audio stream. // a pointer to the popped data is stored as a member in InboundAudioStream. // That's how the popped audio data will be read for mixing (but only if the pop was successful) nodeData->checkBuffersBeforeFrameSend(); // if the stream should be muted, send mute packet if (nodeData->getAvatarAudioStream() && shouldMute(nodeData->getAvatarAudioStream()->getQuietestFrameLoudness())) { auto mutePacket = NLPacket::create(PacketType::NoisyMute, 0); nodeList->sendPacket(std::move(mutePacket), *node); } if (node->getType() == NodeType::Agent && node->getActiveSocket() && nodeData->getAvatarAudioStream()) { int streamsMixed = prepareMixForListeningNode(node.data()); std::unique_ptr<NLPacket> mixPacket; if (streamsMixed > 0) { int mixPacketBytes = sizeof(quint16) + AudioConstants::NETWORK_FRAME_BYTES_STEREO; mixPacket = NLPacket::create(PacketType::MixedAudio, mixPacketBytes); // pack sequence number quint16 sequence = nodeData->getOutgoingSequenceNumber(); mixPacket->writePrimitive(sequence); // pack mixed audio samples mixPacket->write(reinterpret_cast<char*>(_mixSamples), AudioConstants::NETWORK_FRAME_BYTES_STEREO); } else { int silentPacketBytes = sizeof(quint16) + sizeof(quint16); mixPacket = NLPacket::create(PacketType::SilentAudioFrame, silentPacketBytes); // pack sequence number quint16 sequence = nodeData->getOutgoingSequenceNumber(); mixPacket->writePrimitive(sequence); // pack number of silent audio samples quint16 numSilentSamples = AudioConstants::NETWORK_FRAME_SAMPLES_STEREO; mixPacket->writePrimitive(numSilentSamples); } // Send audio environment sendAudioEnvironmentPacket(node); // send mixed audio packet nodeList->sendPacket(std::move(mixPacket), *node); nodeData->incrementOutgoingMixedAudioSequenceNumber(); // send an audio stream stats packet if it's time if (_sendAudioStreamStats) { nodeData->sendAudioStreamStatsPackets(node); _sendAudioStreamStats = false; } ++_sumListeners; } } }); ++_numStatFrames; // since we're a while loop we need to help Qt's event processing QCoreApplication::processEvents(); if (_isFinished) { // at this point the audio-mixer is done // check if we have a deferred delete event to process (which we should once finished) QCoreApplication::sendPostedEvents(this, QEvent::DeferredDelete); break; } usecToSleep = (++nextFrame * AudioConstants::NETWORK_FRAME_USECS) - timer.nsecsElapsed() / 1000; // ns to us if (usecToSleep > 0) { usleep(usecToSleep); } } }
void AudioMixer::broadcastMixes() { auto nodeList = DependencyManager::get<NodeList>(); auto nextFrameTimestamp = p_high_resolution_clock::now(); auto timeToSleep = std::chrono::microseconds(0); const int TRAILING_AVERAGE_FRAMES = 100; int framesSinceCutoffEvent = TRAILING_AVERAGE_FRAMES; int currentFrame { 1 }; int numFramesPerSecond { (int) ceil(AudioConstants::NETWORK_FRAMES_PER_SEC) }; while (!_isFinished) { const float STRUGGLE_TRIGGER_SLEEP_PERCENTAGE_THRESHOLD = 0.10f; const float BACK_OFF_TRIGGER_SLEEP_PERCENTAGE_THRESHOLD = 0.20f; const float RATIO_BACK_OFF = 0.02f; const float CURRENT_FRAME_RATIO = 1.0f / TRAILING_AVERAGE_FRAMES; const float PREVIOUS_FRAMES_RATIO = 1.0f - CURRENT_FRAME_RATIO; if (timeToSleep.count() < 0) { timeToSleep = std::chrono::microseconds(0); } _trailingSleepRatio = (PREVIOUS_FRAMES_RATIO * _trailingSleepRatio) + (timeToSleep.count() * CURRENT_FRAME_RATIO / (float) AudioConstants::NETWORK_FRAME_USECS); float lastCutoffRatio = _performanceThrottlingRatio; bool hasRatioChanged = false; if (framesSinceCutoffEvent >= TRAILING_AVERAGE_FRAMES) { if (_trailingSleepRatio <= STRUGGLE_TRIGGER_SLEEP_PERCENTAGE_THRESHOLD) { // we're struggling - change our min required loudness to reduce some load _performanceThrottlingRatio = _performanceThrottlingRatio + (0.5f * (1.0f - _performanceThrottlingRatio)); qDebug() << "Mixer is struggling, sleeping" << _trailingSleepRatio * 100 << "% of frame time. Old cutoff was" << lastCutoffRatio << "and is now" << _performanceThrottlingRatio; hasRatioChanged = true; } else if (_trailingSleepRatio >= BACK_OFF_TRIGGER_SLEEP_PERCENTAGE_THRESHOLD && _performanceThrottlingRatio != 0) { // we've recovered and can back off the required loudness _performanceThrottlingRatio = _performanceThrottlingRatio - RATIO_BACK_OFF; if (_performanceThrottlingRatio < 0) { _performanceThrottlingRatio = 0; } qDebug() << "Mixer is recovering, sleeping" << _trailingSleepRatio * 100 << "% of frame time. Old cutoff was" << lastCutoffRatio << "and is now" << _performanceThrottlingRatio; hasRatioChanged = true; } if (hasRatioChanged) { // set out min audability threshold from the new ratio _minAudibilityThreshold = LOUDNESS_TO_DISTANCE_RATIO / (2.0f * (1.0f - _performanceThrottlingRatio)); qDebug() << "Minimum audability required to be mixed is now" << _minAudibilityThreshold; framesSinceCutoffEvent = 0; } } if (!hasRatioChanged) { ++framesSinceCutoffEvent; } nodeList->eachNode([&](const SharedNodePointer& node) { if (node->getLinkedData()) { AudioMixerClientData* nodeData = (AudioMixerClientData*)node->getLinkedData(); // this function will attempt to pop a frame from each audio stream. // a pointer to the popped data is stored as a member in InboundAudioStream. // That's how the popped audio data will be read for mixing (but only if the pop was successful) nodeData->checkBuffersBeforeFrameSend(); // if the stream should be muted, send mute packet if (nodeData->getAvatarAudioStream() && shouldMute(nodeData->getAvatarAudioStream()->getQuietestFrameLoudness())) { auto mutePacket = NLPacket::create(PacketType::NoisyMute, 0); nodeList->sendPacket(std::move(mutePacket), *node); } if (node->getType() == NodeType::Agent && node->getActiveSocket() && nodeData->getAvatarAudioStream()) { bool mixHasAudio = prepareMixForListeningNode(node.data()); std::unique_ptr<NLPacket> mixPacket; if (mixHasAudio) { int mixPacketBytes = sizeof(quint16) + AudioConstants::MAX_CODEC_NAME_LENGTH_ON_WIRE + AudioConstants::NETWORK_FRAME_BYTES_STEREO; mixPacket = NLPacket::create(PacketType::MixedAudio, mixPacketBytes); // pack sequence number quint16 sequence = nodeData->getOutgoingSequenceNumber(); mixPacket->writePrimitive(sequence); // write the codec QString codecInPacket = nodeData->getCodecName(); mixPacket->writeString(codecInPacket); QByteArray decodedBuffer(reinterpret_cast<char*>(_clampedSamples), AudioConstants::NETWORK_FRAME_BYTES_STEREO); QByteArray encodedBuffer; nodeData->encode(decodedBuffer, encodedBuffer); // pack mixed audio samples mixPacket->write(encodedBuffer.constData(), encodedBuffer.size()); } else { int silentPacketBytes = sizeof(quint16) + sizeof(quint16) + AudioConstants::MAX_CODEC_NAME_LENGTH_ON_WIRE; mixPacket = NLPacket::create(PacketType::SilentAudioFrame, silentPacketBytes); // pack sequence number quint16 sequence = nodeData->getOutgoingSequenceNumber(); mixPacket->writePrimitive(sequence); // write the codec QString codecInPacket = nodeData->getCodecName(); mixPacket->writeString(codecInPacket); // pack number of silent audio samples quint16 numSilentSamples = AudioConstants::NETWORK_FRAME_SAMPLES_STEREO; mixPacket->writePrimitive(numSilentSamples); } // Send audio environment sendAudioEnvironmentPacket(node); // send mixed audio packet nodeList->sendPacket(std::move(mixPacket), *node); nodeData->incrementOutgoingMixedAudioSequenceNumber(); // send an audio stream stats packet to the client approximately every second ++currentFrame; currentFrame %= numFramesPerSecond; if (nodeData->shouldSendStats(currentFrame)) { nodeData->sendAudioStreamStatsPackets(node); } ++_sumListeners; } } }); ++_numStatFrames; // since we're a while loop we need to help Qt's event processing QCoreApplication::processEvents(); if (_isFinished) { // at this point the audio-mixer is done // check if we have a deferred delete event to process (which we should once finished) QCoreApplication::sendPostedEvents(this, QEvent::DeferredDelete); break; } // push the next frame timestamp to when we should send the next nextFrameTimestamp += std::chrono::microseconds(AudioConstants::NETWORK_FRAME_USECS); // sleep as long as we need until next frame, if we can auto now = p_high_resolution_clock::now(); timeToSleep = std::chrono::duration_cast<std::chrono::microseconds>(nextFrameTimestamp - now); std::this_thread::sleep_for(timeToSleep); } }