void AudioMixer::handleMuteEnvironmentPacket(QSharedPointer<ReceivedMessage> message, SharedNodePointer sendingNode) { auto nodeList = DependencyManager::get<NodeList>(); if (sendingNode->isAllowedEditor()) { glm::vec3 position; float radius; auto newPacket = NLPacket::create(PacketType::MuteEnvironment, sizeof(position) + sizeof(radius)); // read the position and radius from the sent packet message->readPrimitive(&position); message->readPrimitive(&radius); // write them to our packet newPacket->writePrimitive(position); newPacket->writePrimitive(radius); nodeList->eachNode([&](const SharedNodePointer& node){ if (node->getType() == NodeType::Agent && node->getActiveSocket() && node->getLinkedData() && node != sendingNode) { nodeList->sendUnreliablePacket(*newPacket, *node); } }); } }
void AudioMixer::sendStatsPacket() { static QJsonObject statsObject; statsObject["useDynamicJitterBuffers"] = _streamSettings._dynamicJitterBuffers; statsObject["trailing_sleep_percentage"] = _trailingSleepRatio * 100.0f; statsObject["performance_throttling_ratio"] = _performanceThrottlingRatio; statsObject["avg_listeners_per_frame"] = (float) _sumListeners / (float) _numStatFrames; QJsonObject mixStats; mixStats["%_hrtf_mixes"] = percentageForMixStats(_hrtfRenders); mixStats["%_hrtf_silent_mixes"] = percentageForMixStats(_hrtfSilentRenders); mixStats["%_hrtf_struggle_mixes"] = percentageForMixStats(_hrtfStruggleRenders); mixStats["%_manual_stereo_mixes"] = percentageForMixStats(_manualStereoMixes); mixStats["%_manual_echo_mixes"] = percentageForMixStats(_manualEchoMixes); mixStats["total_mixes"] = _totalMixes; mixStats["avg_mixes_per_block"] = _totalMixes / _numStatFrames; statsObject["mix_stats"] = mixStats; _sumListeners = 0; _hrtfRenders = 0; _hrtfSilentRenders = 0; _hrtfStruggleRenders = 0; _manualStereoMixes = 0; _manualEchoMixes = 0; _totalMixes = 0; _numStatFrames = 0; // add stats for each listerner auto nodeList = DependencyManager::get<NodeList>(); QJsonObject listenerStats; nodeList->eachNode([&](const SharedNodePointer& node) { AudioMixerClientData* clientData = static_cast<AudioMixerClientData*>(node->getLinkedData()); if (clientData) { QJsonObject nodeStats; QString uuidString = uuidStringWithoutCurlyBraces(node->getUUID()); nodeStats["outbound_kbps"] = node->getOutboundBandwidth(); nodeStats[USERNAME_UUID_REPLACEMENT_STATS_KEY] = uuidString; nodeStats["jitter"] = clientData->getAudioStreamStats(); listenerStats[uuidString] = nodeStats; } }); // add the listeners object to the root object statsObject["z_listeners"] = listenerStats; // send off the stats packets ThreadedAssignment::addPacketStatsAndSendStatsPacket(statsObject); }
void AudioMixer::handleNodeKilled(SharedNodePointer killedNode) { // enumerate the connected listeners to remove HRTF objects for the disconnected node auto nodeList = DependencyManager::get<NodeList>(); nodeList->eachNode([](const SharedNodePointer& node) { auto clientData = dynamic_cast<AudioMixerClientData*>(node->getLinkedData()); if (clientData) { clientData->removeHRTFsForNode(node->getUUID()); } }); }
void AudioMixer::removeHRTFsForFinishedInjector(const QUuid& streamID) { auto injectorClientData = qobject_cast<AudioMixerClientData*>(sender()); if (injectorClientData) { // enumerate the connected listeners to remove HRTF objects for the disconnected injector auto nodeList = DependencyManager::get<NodeList>(); nodeList->eachNode([injectorClientData, &streamID](const SharedNodePointer& node){ auto listenerClientData = dynamic_cast<AudioMixerClientData*>(node->getLinkedData()); if (listenerClientData) { listenerClientData->removeHRTFForStream(injectorClientData->getNodeID(), streamID); } }); } }
void LimitedNodeList::eraseAllNodes() { qCDebug(networking) << "Clearing the NodeList. Deleting all nodes in list."; QSet<SharedNodePointer> killedNodes; eachNode([&killedNodes](const SharedNodePointer& node) { killedNodes.insert(node); }); // iterate the current nodes, emit that they are dying and remove them from the hash _nodeMutex.lockForWrite(); _nodeHash.clear(); _nodeMutex.unlock(); foreach(const SharedNodePointer& killedNode, killedNodes) { handleNodeKill(killedNode); }
void AudioMixer::handleMuteEnvironmentPacket(QSharedPointer<NLPacket> packet, SharedNodePointer sendingNode) { auto nodeList = DependencyManager::get<NodeList>(); if (sendingNode->getCanAdjustLocks()) { auto newPacket = NLPacket::create(PacketType::MuteEnvironment, packet->getPayloadSize()); // Copy payload newPacket->write(packet->getPayload(), packet->getPayloadSize()); nodeList->eachNode([&](const SharedNodePointer& node){ if (node->getType() == NodeType::Agent && node->getActiveSocket() && node->getLinkedData() && node != sendingNode) { nodeList->sendPacket(std::move(newPacket), *node); } }); } }
// FIXME - make these stats relevant void MessagesMixer::sendStatsPacket() { QJsonObject statsObject; QJsonObject messagesObject; auto nodeList = DependencyManager::get<NodeList>(); // add stats for each listerner nodeList->eachNode([&](const SharedNodePointer& node) { QJsonObject messagesStats; // add the key to ask the domain-server for a username replacement, if it has it messagesStats[USERNAME_UUID_REPLACEMENT_STATS_KEY] = uuidStringWithoutCurlyBraces(node->getUUID()); messagesStats["outbound_kbps"] = node->getOutboundBandwidth(); messagesStats["inbound_kbps"] = node->getInboundBandwidth(); messagesObject[uuidStringWithoutCurlyBraces(node->getUUID())] = messagesStats; }); statsObject["messages"] = messagesObject; ThreadedAssignment::addPacketStatsAndSendStatsPacket(statsObject); }
void AssignmentClientMonitor::checkSpares() { auto nodeList = DependencyManager::get<NodeList>(); QUuid aSpareId = ""; unsigned int spareCount = 0; unsigned int totalCount = 0; nodeList->removeSilentNodes(); nodeList->eachNode([&](const SharedNodePointer& node) { AssignmentClientChildData* childData = static_cast<AssignmentClientChildData*>(node->getLinkedData()); totalCount ++; if (childData->getChildType() == Assignment::Type::AllTypes) { ++spareCount; aSpareId = node->getUUID(); } }); // Spawn or kill children, as needed. If --min or --max weren't specified, allow the child count // to drift up or down as far as needed. if (spareCount < 1 || totalCount < _minAssignmentClientForks) { if (!_maxAssignmentClientForks || totalCount < _maxAssignmentClientForks) { spawnChildClient(); } } if (spareCount > 1) { if (!_minAssignmentClientForks || totalCount > _minAssignmentClientForks) { // kill aSpareId qDebug() << "asking child" << aSpareId << "to exit."; SharedNodePointer childNode = nodeList->nodeWithUUID(aSpareId); childNode->activateLocalSocket(); auto diePacket = NLPacket::create(PacketType::StopNode, 0); nodeList->sendPacket(std::move(diePacket), *childNode); } } }
void ScriptEngine::run() { // TODO: can we add a short circuit for _stoppingAllScripts here? What does it mean to not start running if // we're in the process of stopping? if (!_isInitialized) { init(); } _isRunning = true; _isFinished = false; emit runningStateChanged(); QScriptValue result = evaluate(_scriptContents); QElapsedTimer startTime; startTime.start(); int thisFrame = 0; auto nodeList = DependencyManager::get<NodeList>(); auto entityScriptingInterface = DependencyManager::get<EntityScriptingInterface>(); qint64 lastUpdate = usecTimestampNow(); while (!_isFinished) { int usecToSleep = (thisFrame++ * SCRIPT_DATA_CALLBACK_USECS) - startTime.nsecsElapsed() / 1000; // nsec to usec if (usecToSleep > 0) { usleep(usecToSleep); } if (_isFinished) { break; } QCoreApplication::processEvents(); if (_isFinished) { break; } if (!_isFinished && entityScriptingInterface->getEntityPacketSender()->serversExist()) { // release the queue of edit entity messages. entityScriptingInterface->getEntityPacketSender()->releaseQueuedMessages(); // since we're in non-threaded mode, call process so that the packets are sent if (!entityScriptingInterface->getEntityPacketSender()->isThreaded()) { entityScriptingInterface->getEntityPacketSender()->process(); } } if (!_isFinished && _isAvatar && _avatarData) { const int SCRIPT_AUDIO_BUFFER_SAMPLES = floor(((SCRIPT_DATA_CALLBACK_USECS * AudioConstants::SAMPLE_RATE) / (1000 * 1000)) + 0.5); const int SCRIPT_AUDIO_BUFFER_BYTES = SCRIPT_AUDIO_BUFFER_SAMPLES * sizeof(int16_t); QByteArray avatarByteArray = _avatarData->toByteArray(); auto avatarPacket = NLPacket::create(PacketType::AvatarData, avatarByteArray.size()); avatarPacket->write(avatarByteArray); nodeList->broadcastToNodes(std::move(avatarPacket), NodeSet() << NodeType::AvatarMixer); if (_isListeningToAudioStream || _avatarSound) { // if we have an avatar audio stream then send it out to our audio-mixer bool silentFrame = true; int16_t numAvailableSamples = SCRIPT_AUDIO_BUFFER_SAMPLES; const int16_t* nextSoundOutput = NULL; if (_avatarSound) { const QByteArray& soundByteArray = _avatarSound->getByteArray(); nextSoundOutput = reinterpret_cast<const int16_t*>(soundByteArray.data() + _numAvatarSoundSentBytes); int numAvailableBytes = (soundByteArray.size() - _numAvatarSoundSentBytes) > SCRIPT_AUDIO_BUFFER_BYTES ? SCRIPT_AUDIO_BUFFER_BYTES : soundByteArray.size() - _numAvatarSoundSentBytes; numAvailableSamples = numAvailableBytes / sizeof(int16_t); // check if the all of the _numAvatarAudioBufferSamples to be sent are silence for (int i = 0; i < numAvailableSamples; ++i) { if (nextSoundOutput[i] != 0) { silentFrame = false; break; } } _numAvatarSoundSentBytes += numAvailableBytes; if (_numAvatarSoundSentBytes == soundByteArray.size()) { // we're done with this sound object - so set our pointer back to NULL // and our sent bytes back to zero _avatarSound = NULL; _numAvatarSoundSentBytes = 0; } } auto audioPacket = NLPacket::create(silentFrame ? PacketType::SilentAudioFrame : PacketType::MicrophoneAudioNoEcho); // seek past the sequence number, will be packed when destination node is known audioPacket->seek(sizeof(quint16)); if (silentFrame) { if (!_isListeningToAudioStream) { // if we have a silent frame and we're not listening then just send nothing and break out of here break; } // write the number of silent samples so the audio-mixer can uphold timing audioPacket->writePrimitive(SCRIPT_AUDIO_BUFFER_SAMPLES); // use the orientation and position of this avatar for the source of this audio audioPacket->writePrimitive(_avatarData->getPosition()); glm::quat headOrientation = _avatarData->getHeadOrientation(); audioPacket->writePrimitive(headOrientation); } else if (nextSoundOutput) { // assume scripted avatar audio is mono and set channel flag to zero audioPacket->writePrimitive((quint8) 0); // use the orientation and position of this avatar for the source of this audio audioPacket->writePrimitive(_avatarData->getPosition()); glm::quat headOrientation = _avatarData->getHeadOrientation(); audioPacket->writePrimitive(headOrientation); // write the raw audio data audioPacket->write(reinterpret_cast<const char*>(nextSoundOutput), numAvailableSamples * sizeof(int16_t)); } // write audio packet to AudioMixer nodes auto nodeList = DependencyManager::get<NodeList>(); nodeList->eachNode([this, &nodeList, &audioPacket](const SharedNodePointer& node){ // only send to nodes of type AudioMixer if (node->getType() == NodeType::AudioMixer) { // pack sequence number quint16 sequence = _outgoingScriptAudioSequenceNumbers[node->getUUID()]++; audioPacket->seek(0); audioPacket->writePrimitive(sequence); // send audio packet nodeList->sendUnreliablePacket(*audioPacket, *node); } }); } } qint64 now = usecTimestampNow(); float deltaTime = (float) (now - lastUpdate) / (float) USECS_PER_SECOND; if (hasUncaughtException()) { int line = uncaughtExceptionLineNumber(); qCDebug(scriptengine) << "Uncaught exception at (" << _fileNameString << ") line" << line << ":" << uncaughtException().toString(); emit errorMessage("Uncaught exception at (" + _fileNameString + ") line" + QString::number(line) + ":" + uncaughtException().toString()); clearExceptions(); } if (!_isFinished) { emit update(deltaTime); } lastUpdate = now; } stopAllTimers(); // make sure all our timers are stopped if the script is ending emit scriptEnding(); // kill the avatar identity timer delete _avatarIdentityTimer; if (entityScriptingInterface->getEntityPacketSender()->serversExist()) { // release the queue of edit entity messages. entityScriptingInterface->getEntityPacketSender()->releaseQueuedMessages(); // since we're in non-threaded mode, call process so that the packets are sent if (!entityScriptingInterface->getEntityPacketSender()->isThreaded()) { // wait here till the edit packet sender is completely done sending while (entityScriptingInterface->getEntityPacketSender()->hasPacketsToSend()) { entityScriptingInterface->getEntityPacketSender()->process(); QCoreApplication::processEvents(); } } else { // FIXME - do we need to have a similar "wait here" loop for non-threaded packet senders? } } emit finished(_fileNameString); _isRunning = false; emit runningStateChanged(); emit doneRunning(); _doneRunningThisScript = true; }
void OctreeHeadlessViewer::queryOctree() { char serverType = getMyNodeType(); PacketType packetType = getMyQueryMessageType(); NodeToJurisdictionMap& jurisdictions = *_jurisdictionListener->getJurisdictions(); bool wantExtraDebugging = false; if (wantExtraDebugging) { qCDebug(octree) << "OctreeHeadlessViewer::queryOctree() _jurisdictionListener=" << _jurisdictionListener; qCDebug(octree) << "---------------"; qCDebug(octree) << "_jurisdictionListener=" << _jurisdictionListener; qCDebug(octree) << "Jurisdictions..."; jurisdictions.lockForRead(); for (NodeToJurisdictionMapIterator i = jurisdictions.begin(); i != jurisdictions.end(); ++i) { qCDebug(octree) << i.key() << ": " << &i.value(); } jurisdictions.unlock(); qCDebug(octree) << "---------------"; } // These will be the same for all servers, so we can set them up once and then reuse for each server we send to. _octreeQuery.setWantLowResMoving(true); _octreeQuery.setWantColor(true); _octreeQuery.setWantDelta(true); _octreeQuery.setWantOcclusionCulling(false); _octreeQuery.setWantCompression(true); // TODO: should be on by default _octreeQuery.setCameraPosition(_viewFrustum.getPosition()); _octreeQuery.setCameraOrientation(_viewFrustum.getOrientation()); _octreeQuery.setCameraFov(_viewFrustum.getFieldOfView()); _octreeQuery.setCameraAspectRatio(_viewFrustum.getAspectRatio()); _octreeQuery.setCameraNearClip(_viewFrustum.getNearClip()); _octreeQuery.setCameraFarClip(_viewFrustum.getFarClip()); _octreeQuery.setCameraEyeOffsetPosition(glm::vec3()); _octreeQuery.setOctreeSizeScale(getVoxelSizeScale()); _octreeQuery.setBoundaryLevelAdjust(getBoundaryLevelAdjust()); // Iterate all of the nodes, and get a count of how many voxel servers we have... int totalServers = 0; int inViewServers = 0; int unknownJurisdictionServers = 0; DependencyManager::get<NodeList>()->eachNode([&](const SharedNodePointer& node){ // only send to the NodeTypes that are serverType if (node->getActiveSocket() && node->getType() == serverType) { totalServers++; // get the server bounds for this server QUuid nodeUUID = node->getUUID(); // if we haven't heard from this voxel server, go ahead and send it a query, so we // can get the jurisdiction... jurisdictions.lockForRead(); if (jurisdictions.find(nodeUUID) == jurisdictions.end()) { jurisdictions.unlock(); unknownJurisdictionServers++; } else { const JurisdictionMap& map = (jurisdictions)[nodeUUID]; unsigned char* rootCode = map.getRootOctalCode(); if (rootCode) { VoxelPositionSize rootDetails; voxelDetailsForCode(rootCode, rootDetails); jurisdictions.unlock(); AACube serverBounds(glm::vec3(rootDetails.x, rootDetails.y, rootDetails.z), rootDetails.s); ViewFrustum::location serverFrustumLocation = _viewFrustum.cubeInFrustum(serverBounds); if (serverFrustumLocation != ViewFrustum::OUTSIDE) { inViewServers++; } } else { jurisdictions.unlock(); } } } }); if (wantExtraDebugging) { qCDebug(octree, "Servers: total %d, in view %d, unknown jurisdiction %d", totalServers, inViewServers, unknownJurisdictionServers); } int perServerPPS = 0; const int SMALL_BUDGET = 10; int perUnknownServer = SMALL_BUDGET; int totalPPS = getMaxPacketsPerSecond(); // determine PPS based on number of servers if (inViewServers >= 1) { // set our preferred PPS to be exactly evenly divided among all of the voxel servers... and allocate 1 PPS // for each unknown jurisdiction server perServerPPS = (totalPPS / inViewServers) - (unknownJurisdictionServers * perUnknownServer); } else { if (unknownJurisdictionServers > 0) { perUnknownServer = (totalPPS / unknownJurisdictionServers); } } if (wantExtraDebugging) { qCDebug(octree, "perServerPPS: %d perUnknownServer: %d", perServerPPS, perUnknownServer); } auto nodeList = DependencyManager::get<NodeList>(); nodeList->eachNode([&](const SharedNodePointer& node){ // only send to the NodeTypes that are serverType if (node->getActiveSocket() && node->getType() == serverType) { // get the server bounds for this server QUuid nodeUUID = node->getUUID(); bool inView = false; bool unknownView = false; // if we haven't heard from this voxel server, go ahead and send it a query, so we // can get the jurisdiction... jurisdictions.lockForRead(); if (jurisdictions.find(nodeUUID) == jurisdictions.end()) { jurisdictions.unlock(); unknownView = true; // assume it's in view if (wantExtraDebugging) { qCDebug(octree) << "no known jurisdiction for node " << *node << ", assume it's visible."; } } else { const JurisdictionMap& map = (jurisdictions)[nodeUUID]; unsigned char* rootCode = map.getRootOctalCode(); if (rootCode) { VoxelPositionSize rootDetails; voxelDetailsForCode(rootCode, rootDetails); jurisdictions.unlock(); AACube serverBounds(glm::vec3(rootDetails.x, rootDetails.y, rootDetails.z), rootDetails.s); ViewFrustum::location serverFrustumLocation = _viewFrustum.cubeInFrustum(serverBounds); if (serverFrustumLocation != ViewFrustum::OUTSIDE) { inView = true; } else { inView = false; } } else { jurisdictions.unlock(); if (wantExtraDebugging) { qCDebug(octree) << "Jurisdiction without RootCode for node " << *node << ". That's unusual!"; } } } if (inView) { _octreeQuery.setMaxQueryPacketsPerSecond(perServerPPS); if (wantExtraDebugging) { qCDebug(octree) << "inView for node " << *node << ", give it budget of " << perServerPPS; } } else if (unknownView) { if (wantExtraDebugging) { qCDebug(octree) << "no known jurisdiction for node " << *node << ", give it budget of " << perUnknownServer << " to send us jurisdiction."; } // set the query's position/orientation to be degenerate in a manner that will get the scene quickly // If there's only one server, then don't do this, and just let the normal voxel query pass through // as expected... this way, we will actually get a valid scene if there is one to be seen if (totalServers > 1) { _octreeQuery.setCameraPosition(glm::vec3(-0.1,-0.1,-0.1)); const glm::quat OFF_IN_NEGATIVE_SPACE = glm::quat(-0.5, 0, -0.5, 1.0); _octreeQuery.setCameraOrientation(OFF_IN_NEGATIVE_SPACE); _octreeQuery.setCameraNearClip(0.1f); _octreeQuery.setCameraFarClip(0.1f); if (wantExtraDebugging) { qCDebug(octree) << "Using 'minimal' camera position for node" << *node; } } else { if (wantExtraDebugging) { qCDebug(octree) << "Using regular camera position for node" << *node; } } _octreeQuery.setMaxQueryPacketsPerSecond(perUnknownServer); } else { _octreeQuery.setMaxQueryPacketsPerSecond(0); } // setup the query packet auto queryPacket = NLPacket::create(packetType); _octreeQuery.getBroadcastData(reinterpret_cast<unsigned char*>(queryPacket->getPayload())); // ask the NodeList to send it nodeList->sendPacket(std::move(queryPacket), *node); } }); }
void AudioMixer::run() { ThreadedAssignment::commonInit(AUDIO_MIXER_LOGGING_TARGET_NAME, NodeType::AudioMixer); auto nodeList = DependencyManager::get<NodeList>(); nodeList->addNodeTypeToInterestSet(NodeType::Agent); nodeList->linkedDataCreateCallback = [](Node* node) { node->setLinkedData(new AudioMixerClientData()); }; // wait until we have the domain-server settings, otherwise we bail DomainHandler& domainHandler = nodeList->getDomainHandler(); qDebug() << "Waiting for domain settings from domain-server."; // block until we get the settingsRequestComplete signal QEventLoop loop; connect(&domainHandler, &DomainHandler::settingsReceived, &loop, &QEventLoop::quit); connect(&domainHandler, &DomainHandler::settingsReceiveFail, &loop, &QEventLoop::quit); domainHandler.requestDomainSettings(); loop.exec(); if (domainHandler.getSettingsObject().isEmpty()) { qDebug() << "Failed to retreive settings object from domain-server. Bailing on assignment."; setFinished(true); return; } const QJsonObject& settingsObject = domainHandler.getSettingsObject(); // check the settings object to see if we have anything we can parse out parseSettingsObject(settingsObject); int nextFrame = 0; QElapsedTimer timer; timer.start(); int usecToSleep = AudioConstants::NETWORK_FRAME_USECS; const int TRAILING_AVERAGE_FRAMES = 100; int framesSinceCutoffEvent = TRAILING_AVERAGE_FRAMES; while (!_isFinished) { const float STRUGGLE_TRIGGER_SLEEP_PERCENTAGE_THRESHOLD = 0.10f; const float BACK_OFF_TRIGGER_SLEEP_PERCENTAGE_THRESHOLD = 0.20f; const float RATIO_BACK_OFF = 0.02f; const float CURRENT_FRAME_RATIO = 1.0f / TRAILING_AVERAGE_FRAMES; const float PREVIOUS_FRAMES_RATIO = 1.0f - CURRENT_FRAME_RATIO; if (usecToSleep < 0) { usecToSleep = 0; } _trailingSleepRatio = (PREVIOUS_FRAMES_RATIO * _trailingSleepRatio) + (usecToSleep * CURRENT_FRAME_RATIO / (float) AudioConstants::NETWORK_FRAME_USECS); float lastCutoffRatio = _performanceThrottlingRatio; bool hasRatioChanged = false; if (framesSinceCutoffEvent >= TRAILING_AVERAGE_FRAMES) { if (_trailingSleepRatio <= STRUGGLE_TRIGGER_SLEEP_PERCENTAGE_THRESHOLD) { // we're struggling - change our min required loudness to reduce some load _performanceThrottlingRatio = _performanceThrottlingRatio + (0.5f * (1.0f - _performanceThrottlingRatio)); qDebug() << "Mixer is struggling, sleeping" << _trailingSleepRatio * 100 << "% of frame time. Old cutoff was" << lastCutoffRatio << "and is now" << _performanceThrottlingRatio; hasRatioChanged = true; } else if (_trailingSleepRatio >= BACK_OFF_TRIGGER_SLEEP_PERCENTAGE_THRESHOLD && _performanceThrottlingRatio != 0) { // we've recovered and can back off the required loudness _performanceThrottlingRatio = _performanceThrottlingRatio - RATIO_BACK_OFF; if (_performanceThrottlingRatio < 0) { _performanceThrottlingRatio = 0; } qDebug() << "Mixer is recovering, sleeping" << _trailingSleepRatio * 100 << "% of frame time. Old cutoff was" << lastCutoffRatio << "and is now" << _performanceThrottlingRatio; hasRatioChanged = true; } if (hasRatioChanged) { // set out min audability threshold from the new ratio _minAudibilityThreshold = LOUDNESS_TO_DISTANCE_RATIO / (2.0f * (1.0f - _performanceThrottlingRatio)); qDebug() << "Minimum audability required to be mixed is now" << _minAudibilityThreshold; framesSinceCutoffEvent = 0; } } if (!hasRatioChanged) { ++framesSinceCutoffEvent; } quint64 now = usecTimestampNow(); if (now - _lastPerSecondCallbackTime > USECS_PER_SECOND) { perSecondActions(); _lastPerSecondCallbackTime = now; } nodeList->eachNode([&](const SharedNodePointer& node) { if (node->getLinkedData()) { AudioMixerClientData* nodeData = (AudioMixerClientData*)node->getLinkedData(); // this function will attempt to pop a frame from each audio stream. // a pointer to the popped data is stored as a member in InboundAudioStream. // That's how the popped audio data will be read for mixing (but only if the pop was successful) nodeData->checkBuffersBeforeFrameSend(); // if the stream should be muted, send mute packet if (nodeData->getAvatarAudioStream() && shouldMute(nodeData->getAvatarAudioStream()->getQuietestFrameLoudness())) { auto mutePacket = NLPacket::create(PacketType::NoisyMute, 0); nodeList->sendPacket(std::move(mutePacket), *node); } if (node->getType() == NodeType::Agent && node->getActiveSocket() && nodeData->getAvatarAudioStream()) { int streamsMixed = prepareMixForListeningNode(node.data()); std::unique_ptr<NLPacket> mixPacket; if (streamsMixed > 0) { int mixPacketBytes = sizeof(quint16) + AudioConstants::NETWORK_FRAME_BYTES_STEREO; mixPacket = NLPacket::create(PacketType::MixedAudio, mixPacketBytes); // pack sequence number quint16 sequence = nodeData->getOutgoingSequenceNumber(); mixPacket->writePrimitive(sequence); // pack mixed audio samples mixPacket->write(reinterpret_cast<char*>(_mixSamples), AudioConstants::NETWORK_FRAME_BYTES_STEREO); } else { int silentPacketBytes = sizeof(quint16) + sizeof(quint16); mixPacket = NLPacket::create(PacketType::SilentAudioFrame, silentPacketBytes); // pack sequence number quint16 sequence = nodeData->getOutgoingSequenceNumber(); mixPacket->writePrimitive(sequence); // pack number of silent audio samples quint16 numSilentSamples = AudioConstants::NETWORK_FRAME_SAMPLES_STEREO; mixPacket->writePrimitive(numSilentSamples); } // Send audio environment sendAudioEnvironmentPacket(node); // send mixed audio packet nodeList->sendPacket(std::move(mixPacket), *node); nodeData->incrementOutgoingMixedAudioSequenceNumber(); // send an audio stream stats packet if it's time if (_sendAudioStreamStats) { nodeData->sendAudioStreamStatsPackets(node); _sendAudioStreamStats = false; } ++_sumListeners; } } }); ++_numStatFrames; // since we're a while loop we need to help Qt's event processing QCoreApplication::processEvents(); if (_isFinished) { // at this point the audio-mixer is done // check if we have a deferred delete event to process (which we should once finished) QCoreApplication::sendPostedEvents(this, QEvent::DeferredDelete); break; } usecToSleep = (++nextFrame * AudioConstants::NETWORK_FRAME_USECS) - timer.nsecsElapsed() / 1000; // ns to us if (usecToSleep > 0) { usleep(usecToSleep); } } }
void AudioMixer::sendStatsPacket() { static QJsonObject statsObject; statsObject["useDynamicJitterBuffers"] = _streamSettings._dynamicJitterBuffers; statsObject["trailing_sleep_percentage"] = _trailingSleepRatio * 100.0f; statsObject["performance_throttling_ratio"] = _performanceThrottlingRatio; statsObject["average_listeners_per_frame"] = (float) _sumListeners / (float) _numStatFrames; if (_sumListeners > 0) { statsObject["average_mixes_per_listener"] = (float) _sumMixes / (float) _sumListeners; } else { statsObject["average_mixes_per_listener"] = 0.0; } _sumListeners = 0; _sumMixes = 0; _numStatFrames = 0; QJsonObject readPendingDatagramStats; QJsonObject rpdCallsStats; rpdCallsStats["calls_per_sec_avg_30s"] = _readPendingCallsPerSecondStats.getWindowAverage(); rpdCallsStats["calls_last_sec"] = _readPendingCallsPerSecondStats.getLastCompleteIntervalStats().getSum() + 0.5; readPendingDatagramStats["calls"] = rpdCallsStats; QJsonObject packetsPerCallStats; packetsPerCallStats["avg_30s"] = _datagramsReadPerCallStats.getWindowAverage(); packetsPerCallStats["avg_1s"] = _datagramsReadPerCallStats.getLastCompleteIntervalStats().getAverage(); readPendingDatagramStats["packets_per_call"] = packetsPerCallStats; QJsonObject packetsTimePerCallStats; packetsTimePerCallStats["usecs_per_call_avg_30s"] = _timeSpentPerCallStats.getWindowAverage(); packetsTimePerCallStats["usecs_per_call_avg_1s"] = _timeSpentPerCallStats.getLastCompleteIntervalStats().getAverage(); packetsTimePerCallStats["prct_time_in_call_30s"] = _timeSpentPerCallStats.getWindowSum() / (READ_DATAGRAMS_STATS_WINDOW_SECONDS * USECS_PER_SECOND) * 100.0; packetsTimePerCallStats["prct_time_in_call_1s"] = _timeSpentPerCallStats.getLastCompleteIntervalStats().getSum() / USECS_PER_SECOND * 100.0; readPendingDatagramStats["packets_time_per_call"] = packetsTimePerCallStats; QJsonObject hashMatchTimePerCallStats; hashMatchTimePerCallStats["usecs_per_hashmatch_avg_30s"] = _timeSpentPerHashMatchCallStats.getWindowAverage(); hashMatchTimePerCallStats["usecs_per_hashmatch_avg_1s"] = _timeSpentPerHashMatchCallStats.getLastCompleteIntervalStats().getAverage(); hashMatchTimePerCallStats["prct_time_in_hashmatch_30s"] = _timeSpentPerHashMatchCallStats.getWindowSum() / (READ_DATAGRAMS_STATS_WINDOW_SECONDS*USECS_PER_SECOND) * 100.0; hashMatchTimePerCallStats["prct_time_in_hashmatch_1s"] = _timeSpentPerHashMatchCallStats.getLastCompleteIntervalStats().getSum() / USECS_PER_SECOND * 100.0; readPendingDatagramStats["hashmatch_time_per_call"] = hashMatchTimePerCallStats; statsObject["read_pending_datagrams"] = readPendingDatagramStats; // add stats for each listerner auto nodeList = DependencyManager::get<NodeList>(); QJsonObject listenerStats; nodeList->eachNode([&](const SharedNodePointer& node) { AudioMixerClientData* clientData = static_cast<AudioMixerClientData*>(node->getLinkedData()); if (clientData) { QJsonObject nodeStats; QString uuidString = uuidStringWithoutCurlyBraces(node->getUUID()); nodeStats["outbound_kbps"] = node->getOutboundBandwidth(); nodeStats[USERNAME_UUID_REPLACEMENT_STATS_KEY] = uuidString; nodeStats["jitter"] = clientData->getAudioStreamStats(); listenerStats[uuidString] = nodeStats; } }); // add the listeners object to the root object statsObject["listeners"] = listenerStats; // send off the stats packets ThreadedAssignment::addPacketStatsAndSendStatsPacket(statsObject); }
void NodeBounds::draw() { if (!_showEntityNodes) { _overlayText[0] = '\0'; return; } NodeToJurisdictionMap& entityServerJurisdictions = Application::getInstance()->getEntityServerJurisdictions(); NodeToJurisdictionMap* serverJurisdictions; // Compute ray to find selected nodes later on. We can't use the pre-computed ray in Application because it centers // itself after the cursor disappears. Application* application = Application::getInstance(); PickRay pickRay = application->getCamera()->computePickRay(application->getTrueMouseX(), application->getTrueMouseY()); // Variables to keep track of the selected node and properties to draw the cube later if needed Node* selectedNode = NULL; float selectedDistance = FLT_MAX; bool selectedIsInside = true; glm::vec3 selectedCenter; float selectedScale = 0; auto nodeList = DependencyManager::get<NodeList>(); nodeList->eachNode([&](const SharedNodePointer& node){ NodeType_t nodeType = node->getType(); if (nodeType == NodeType::EntityServer && _showEntityNodes) { serverJurisdictions = &entityServerJurisdictions; } else { return; } QUuid nodeUUID = node->getUUID(); serverJurisdictions->lockForRead(); if (serverJurisdictions->find(nodeUUID) != serverJurisdictions->end()) { const JurisdictionMap& map = (*serverJurisdictions)[nodeUUID]; unsigned char* rootCode = map.getRootOctalCode(); if (rootCode) { VoxelPositionSize rootDetails; voxelDetailsForCode(rootCode, rootDetails); serverJurisdictions->unlock(); glm::vec3 location(rootDetails.x, rootDetails.y, rootDetails.z); AACube serverBounds(location, rootDetails.s); glm::vec3 center = serverBounds.getVertex(BOTTOM_RIGHT_NEAR) + ((serverBounds.getVertex(TOP_LEFT_FAR) - serverBounds.getVertex(BOTTOM_RIGHT_NEAR)) / 2.0f); const float ENTITY_NODE_SCALE = 0.99f; float scaleFactor = rootDetails.s; // Scale by 0.92 - 1.00 depending on the scale of the node. This allows smaller nodes to scale in // a bit and not overlap larger nodes. scaleFactor *= 0.92f + (rootDetails.s * 0.08f); // Scale different node types slightly differently because it's common for them to overlap. if (nodeType == NodeType::EntityServer) { scaleFactor *= ENTITY_NODE_SCALE; } float red, green, blue; getColorForNodeType(nodeType, red, green, blue); drawNodeBorder(center, scaleFactor, red, green, blue); float distance; BoxFace face; bool inside = serverBounds.contains(pickRay.origin); bool colliding = serverBounds.findRayIntersection(pickRay.origin, pickRay.direction, distance, face); // If the camera is inside a node it will be "selected" if you don't have your cursor over another node // that you aren't inside. if (colliding && (!selectedNode || (!inside && (distance < selectedDistance || selectedIsInside)))) { selectedNode = node.data(); selectedDistance = distance; selectedIsInside = inside; selectedCenter = center; selectedScale = scaleFactor; } } else { serverJurisdictions->unlock(); } } else { serverJurisdictions->unlock(); } }); if (selectedNode) { glPushMatrix(); glTranslatef(selectedCenter.x, selectedCenter.y, selectedCenter.z); glScalef(selectedScale, selectedScale, selectedScale); float red, green, blue; getColorForNodeType(selectedNode->getType(), red, green, blue); DependencyManager::get<GeometryCache>()->renderSolidCube(1.0f, glm::vec4(red, green, blue, 0.2f)); glPopMatrix(); HifiSockAddr addr = selectedNode->getPublicSocket(); QString overlay = QString("%1:%2 %3ms") .arg(addr.getAddress().toString()) .arg(addr.getPort()) .arg(selectedNode->getPingMs()) .left(MAX_OVERLAY_TEXT_LENGTH); // Ideally we'd just use a QString, but I ran into weird blinking issues using // constData() directly, as if the data was being overwritten. strcpy(_overlayText, overlay.toLocal8Bit().constData()); } else { _overlayText[0] = '\0'; } }
void AudioMixer::broadcastMixes() { auto nodeList = DependencyManager::get<NodeList>(); auto nextFrameTimestamp = p_high_resolution_clock::now(); auto timeToSleep = std::chrono::microseconds(0); const int TRAILING_AVERAGE_FRAMES = 100; int framesSinceCutoffEvent = TRAILING_AVERAGE_FRAMES; int currentFrame { 1 }; int numFramesPerSecond { (int) ceil(AudioConstants::NETWORK_FRAMES_PER_SEC) }; while (!_isFinished) { const float STRUGGLE_TRIGGER_SLEEP_PERCENTAGE_THRESHOLD = 0.10f; const float BACK_OFF_TRIGGER_SLEEP_PERCENTAGE_THRESHOLD = 0.20f; const float RATIO_BACK_OFF = 0.02f; const float CURRENT_FRAME_RATIO = 1.0f / TRAILING_AVERAGE_FRAMES; const float PREVIOUS_FRAMES_RATIO = 1.0f - CURRENT_FRAME_RATIO; if (timeToSleep.count() < 0) { timeToSleep = std::chrono::microseconds(0); } _trailingSleepRatio = (PREVIOUS_FRAMES_RATIO * _trailingSleepRatio) + (timeToSleep.count() * CURRENT_FRAME_RATIO / (float) AudioConstants::NETWORK_FRAME_USECS); float lastCutoffRatio = _performanceThrottlingRatio; bool hasRatioChanged = false; if (framesSinceCutoffEvent >= TRAILING_AVERAGE_FRAMES) { if (_trailingSleepRatio <= STRUGGLE_TRIGGER_SLEEP_PERCENTAGE_THRESHOLD) { // we're struggling - change our min required loudness to reduce some load _performanceThrottlingRatio = _performanceThrottlingRatio + (0.5f * (1.0f - _performanceThrottlingRatio)); qDebug() << "Mixer is struggling, sleeping" << _trailingSleepRatio * 100 << "% of frame time. Old cutoff was" << lastCutoffRatio << "and is now" << _performanceThrottlingRatio; hasRatioChanged = true; } else if (_trailingSleepRatio >= BACK_OFF_TRIGGER_SLEEP_PERCENTAGE_THRESHOLD && _performanceThrottlingRatio != 0) { // we've recovered and can back off the required loudness _performanceThrottlingRatio = _performanceThrottlingRatio - RATIO_BACK_OFF; if (_performanceThrottlingRatio < 0) { _performanceThrottlingRatio = 0; } qDebug() << "Mixer is recovering, sleeping" << _trailingSleepRatio * 100 << "% of frame time. Old cutoff was" << lastCutoffRatio << "and is now" << _performanceThrottlingRatio; hasRatioChanged = true; } if (hasRatioChanged) { // set out min audability threshold from the new ratio _minAudibilityThreshold = LOUDNESS_TO_DISTANCE_RATIO / (2.0f * (1.0f - _performanceThrottlingRatio)); qDebug() << "Minimum audability required to be mixed is now" << _minAudibilityThreshold; framesSinceCutoffEvent = 0; } } if (!hasRatioChanged) { ++framesSinceCutoffEvent; } nodeList->eachNode([&](const SharedNodePointer& node) { if (node->getLinkedData()) { AudioMixerClientData* nodeData = (AudioMixerClientData*)node->getLinkedData(); // this function will attempt to pop a frame from each audio stream. // a pointer to the popped data is stored as a member in InboundAudioStream. // That's how the popped audio data will be read for mixing (but only if the pop was successful) nodeData->checkBuffersBeforeFrameSend(); // if the stream should be muted, send mute packet if (nodeData->getAvatarAudioStream() && shouldMute(nodeData->getAvatarAudioStream()->getQuietestFrameLoudness())) { auto mutePacket = NLPacket::create(PacketType::NoisyMute, 0); nodeList->sendPacket(std::move(mutePacket), *node); } if (node->getType() == NodeType::Agent && node->getActiveSocket() && nodeData->getAvatarAudioStream()) { bool mixHasAudio = prepareMixForListeningNode(node.data()); std::unique_ptr<NLPacket> mixPacket; if (mixHasAudio) { int mixPacketBytes = sizeof(quint16) + AudioConstants::MAX_CODEC_NAME_LENGTH_ON_WIRE + AudioConstants::NETWORK_FRAME_BYTES_STEREO; mixPacket = NLPacket::create(PacketType::MixedAudio, mixPacketBytes); // pack sequence number quint16 sequence = nodeData->getOutgoingSequenceNumber(); mixPacket->writePrimitive(sequence); // write the codec QString codecInPacket = nodeData->getCodecName(); mixPacket->writeString(codecInPacket); QByteArray decodedBuffer(reinterpret_cast<char*>(_clampedSamples), AudioConstants::NETWORK_FRAME_BYTES_STEREO); QByteArray encodedBuffer; nodeData->encode(decodedBuffer, encodedBuffer); // pack mixed audio samples mixPacket->write(encodedBuffer.constData(), encodedBuffer.size()); } else { int silentPacketBytes = sizeof(quint16) + sizeof(quint16) + AudioConstants::MAX_CODEC_NAME_LENGTH_ON_WIRE; mixPacket = NLPacket::create(PacketType::SilentAudioFrame, silentPacketBytes); // pack sequence number quint16 sequence = nodeData->getOutgoingSequenceNumber(); mixPacket->writePrimitive(sequence); // write the codec QString codecInPacket = nodeData->getCodecName(); mixPacket->writeString(codecInPacket); // pack number of silent audio samples quint16 numSilentSamples = AudioConstants::NETWORK_FRAME_SAMPLES_STEREO; mixPacket->writePrimitive(numSilentSamples); } // Send audio environment sendAudioEnvironmentPacket(node); // send mixed audio packet nodeList->sendPacket(std::move(mixPacket), *node); nodeData->incrementOutgoingMixedAudioSequenceNumber(); // send an audio stream stats packet to the client approximately every second ++currentFrame; currentFrame %= numFramesPerSecond; if (nodeData->shouldSendStats(currentFrame)) { nodeData->sendAudioStreamStatsPackets(node); } ++_sumListeners; } } }); ++_numStatFrames; // since we're a while loop we need to help Qt's event processing QCoreApplication::processEvents(); if (_isFinished) { // at this point the audio-mixer is done // check if we have a deferred delete event to process (which we should once finished) QCoreApplication::sendPostedEvents(this, QEvent::DeferredDelete); break; } // push the next frame timestamp to when we should send the next nextFrameTimestamp += std::chrono::microseconds(AudioConstants::NETWORK_FRAME_USECS); // sleep as long as we need until next frame, if we can auto now = p_high_resolution_clock::now(); timeToSleep = std::chrono::duration_cast<std::chrono::microseconds>(nextFrameTimestamp - now); std::this_thread::sleep_for(timeToSleep); } }