void AudioMixerClientData::sendAudioStreamStatsPackets(const SharedNodePointer& destinationNode) {
// have all the seq number stats of each audio stream push their current stats into their history,
// which moves that history window 1 second forward (since that's how long since the last stats were pushed into history)
_incomingAvatarAudioSequenceNumberStats.pushStatsToHistory();
QHash<QUuid, SequenceNumberStats>::Iterator i = _incomingInjectedAudioSequenceNumberStatsMap.begin();
QHash<QUuid, SequenceNumberStats>::Iterator end = _incomingInjectedAudioSequenceNumberStatsMap.end();
while (i != end) {
i.value().pushStatsToHistory();
i++;
}
char packet[MAX_PACKET_SIZE];
NodeList* nodeList = NodeList::getInstance();
// The append flag is a boolean value that will be packed right after the header. The first packet sent
// inside this method will have 0 for this flag, while every subsequent packet will have 1 for this flag.
// The sole purpose of this flag is so the client can clear its map of injected audio stream stats when
// it receives a packet with an appendFlag of 0. This prevents the buildup of dead audio stream stats in the client.
quint8 appendFlag = 0;
// pack header
int numBytesPacketHeader = populatePacketHeader(packet, PacketTypeAudioStreamStats);
char* headerEndAt = packet + numBytesPacketHeader;
// calculate how many stream stat structs we can fit in each packet
const int numStreamStatsRoomFor = (MAX_PACKET_SIZE - numBytesPacketHeader - sizeof(quint8) - sizeof(quint16)) / sizeof(AudioStreamStats);
// pack and send stream stats packets until all ring buffers' stats are sent
int numStreamStatsRemaining = _ringBuffers.size();
QList<PositionalAudioRingBuffer*>::ConstIterator ringBuffersIterator = _ringBuffers.constBegin();
while (numStreamStatsRemaining > 0) {
char* dataAt = headerEndAt;
// pack the append flag
memcpy(dataAt, &appendFlag, sizeof(quint8));
appendFlag = 1;
dataAt += sizeof(quint8);
// calculate and pack the number of stream stats to follow
quint16 numStreamStatsToPack = std::min(numStreamStatsRemaining, numStreamStatsRoomFor);
memcpy(dataAt, &numStreamStatsToPack, sizeof(quint16));
dataAt += sizeof(quint16);
// pack the calculated number of stream stats
for (int i = 0; i < numStreamStatsToPack; i++) {
AudioStreamStats streamStats = getAudioStreamStatsOfStream(*ringBuffersIterator);
memcpy(dataAt, &streamStats, sizeof(AudioStreamStats));
dataAt += sizeof(AudioStreamStats);
ringBuffersIterator++;
}
numStreamStatsRemaining -= numStreamStatsToPack;
// send the current packet
nodeList->writeDatagram(packet, dataAt - packet, destinationNode);
}
}
void AudioMixer::readPendingDatagram(const QByteArray& receivedPacket, const HifiSockAddr& senderSockAddr) {
NodeList* nodeList = NodeList::getInstance();
if (nodeList->packetVersionAndHashMatch(receivedPacket)) {
// pull any new audio data from nodes off of the network stack
PacketType mixerPacketType = packetTypeForPacket(receivedPacket);
if (mixerPacketType == PacketTypeMicrophoneAudioNoEcho
|| mixerPacketType == PacketTypeMicrophoneAudioWithEcho
|| mixerPacketType == PacketTypeInjectAudio
|| mixerPacketType == PacketTypeSilentAudioFrame
|| mixerPacketType == PacketTypeAudioStreamStats) {
nodeList->findNodeAndUpdateWithDataFromPacket(receivedPacket);
} else if (mixerPacketType == PacketTypeMuteEnvironment) {
QByteArray packet = receivedPacket;
populatePacketHeader(packet, PacketTypeMuteEnvironment);
foreach (const SharedNodePointer& node, nodeList->getNodeHash()) {
if (node->getType() == NodeType::Agent && node->getActiveSocket() && node->getLinkedData() && node != nodeList->sendingNodeForPacket(receivedPacket)) {
nodeList->writeDatagram(packet, packet.size(), node);
}
}
} else {
void AudioInjector::injectAudio() {
QByteArray soundByteArray = _sound->getByteArray();
// make sure we actually have samples downloaded to inject
if (soundByteArray.size()) {
// give our sample byte array to the local audio interface, if we have it, so it can be handled locally
if (_options.getLoopbackAudioInterface()) {
// assume that localAudioInterface could be on a separate thread, use Qt::AutoConnection to handle properly
QMetaObject::invokeMethod(_options.getLoopbackAudioInterface(), "handleAudioByteArray",
Qt::AutoConnection,
Q_ARG(QByteArray, soundByteArray));
}
NodeList* nodeList = NodeList::getInstance();
// setup the packet for injected audio
QByteArray injectAudioPacket = byteArrayWithPopulatedHeader(PacketTypeInjectAudio);
QDataStream packetStream(&injectAudioPacket, QIODevice::Append);
packetStream << QUuid::createUuid();
// pack the flag for loopback
uchar loopbackFlag = (uchar) (!_options.getLoopbackAudioInterface());
packetStream << loopbackFlag;
// pack the position for injected audio
packetStream.writeRawData(reinterpret_cast<const char*>(&_options.getPosition()), sizeof(_options.getPosition()));
// pack our orientation for injected audio
packetStream.writeRawData(reinterpret_cast<const char*>(&_options.getOrientation()), sizeof(_options.getOrientation()));
// pack zero for radius
float radius = 0;
packetStream << radius;
// pack 255 for attenuation byte
quint8 volume = MAX_INJECTOR_VOLUME * _options.getVolume();
packetStream << volume;
QElapsedTimer timer;
timer.start();
int nextFrame = 0;
int currentSendPosition = 0;
int numPreAudioDataBytes = injectAudioPacket.size();
// loop to send off our audio in NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL byte chunks
while (currentSendPosition < soundByteArray.size() && !_shouldStop) {
int bytesToCopy = std::min(NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL,
soundByteArray.size() - currentSendPosition);
// resize the QByteArray to the right size
injectAudioPacket.resize(numPreAudioDataBytes + bytesToCopy);
// copy the next NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL bytes to the packet
memcpy(injectAudioPacket.data() + numPreAudioDataBytes, soundByteArray.data() + currentSendPosition, bytesToCopy);
// grab our audio mixer from the NodeList, if it exists
SharedNodePointer audioMixer = nodeList->soloNodeOfType(NodeType::AudioMixer);
// send off this audio packet
nodeList->writeDatagram(injectAudioPacket, audioMixer);
currentSendPosition += bytesToCopy;
// send two packets before the first sleep so the mixer can start playback right away
if (currentSendPosition != bytesToCopy && currentSendPosition < soundByteArray.size()) {
// not the first packet and not done
// sleep for the appropriate time
int usecToSleep = (++nextFrame * BUFFER_SEND_INTERVAL_USECS) - timer.nsecsElapsed() / 1000;
if (usecToSleep > 0) {
usleep(usecToSleep);
}
}
}
}
emit finished();
}
// NOTE: some additional optimizations to consider.
// 1) use the view frustum to cull those avatars that are out of view. Since avatar data doesn't need to be present
// if the avatar is not in view or in the keyhole.
void AvatarMixer::broadcastAvatarData() {
int idleTime = QDateTime::currentMSecsSinceEpoch() - _lastFrameTimestamp;
++_numStatFrames;
const float STRUGGLE_TRIGGER_SLEEP_PERCENTAGE_THRESHOLD = 0.10f;
const float BACK_OFF_TRIGGER_SLEEP_PERCENTAGE_THRESHOLD = 0.20f;
const float RATIO_BACK_OFF = 0.02f;
const int TRAILING_AVERAGE_FRAMES = 100;
int framesSinceCutoffEvent = TRAILING_AVERAGE_FRAMES;
const float CURRENT_FRAME_RATIO = 1.0f / TRAILING_AVERAGE_FRAMES;
const float PREVIOUS_FRAMES_RATIO = 1.0f - CURRENT_FRAME_RATIO;
_trailingSleepRatio = (PREVIOUS_FRAMES_RATIO * _trailingSleepRatio)
+ (idleTime * CURRENT_FRAME_RATIO / (float) AVATAR_DATA_SEND_INTERVAL_MSECS);
float lastCutoffRatio = _performanceThrottlingRatio;
bool hasRatioChanged = false;
if (framesSinceCutoffEvent >= TRAILING_AVERAGE_FRAMES) {
if (_trailingSleepRatio <= STRUGGLE_TRIGGER_SLEEP_PERCENTAGE_THRESHOLD) {
// we're struggling - change our min required loudness to reduce some load
_performanceThrottlingRatio = _performanceThrottlingRatio + (0.5f * (1.0f - _performanceThrottlingRatio));
qDebug() << "Mixer is struggling, sleeping" << _trailingSleepRatio * 100 << "% of frame time. Old cutoff was"
<< lastCutoffRatio << "and is now" << _performanceThrottlingRatio;
hasRatioChanged = true;
} else if (_trailingSleepRatio >= BACK_OFF_TRIGGER_SLEEP_PERCENTAGE_THRESHOLD && _performanceThrottlingRatio != 0) {
// we've recovered and can back off the required loudness
_performanceThrottlingRatio = _performanceThrottlingRatio - RATIO_BACK_OFF;
if (_performanceThrottlingRatio < 0) {
_performanceThrottlingRatio = 0;
}
qDebug() << "Mixer is recovering, sleeping" << _trailingSleepRatio * 100 << "% of frame time. Old cutoff was"
<< lastCutoffRatio << "and is now" << _performanceThrottlingRatio;
hasRatioChanged = true;
}
if (hasRatioChanged) {
framesSinceCutoffEvent = 0;
}
}
if (!hasRatioChanged) {
++framesSinceCutoffEvent;
}
static QByteArray mixedAvatarByteArray;
int numPacketHeaderBytes = populatePacketHeader(mixedAvatarByteArray, PacketTypeBulkAvatarData);
NodeList* nodeList = NodeList::getInstance();
AvatarMixerClientData* nodeData = NULL;
AvatarMixerClientData* otherNodeData = NULL;
foreach (const SharedNodePointer& node, nodeList->getNodeHash()) {
if (node->getLinkedData() && node->getType() == NodeType::Agent && node->getActiveSocket()
&& (nodeData = reinterpret_cast<AvatarMixerClientData*>(node->getLinkedData()))->getMutex().tryLock()) {
++_sumListeners;
// reset packet pointers for this node
mixedAvatarByteArray.resize(numPacketHeaderBytes);
AvatarData& avatar = nodeData->getAvatar();
glm::vec3 myPosition = avatar.getPosition();
// this is an AGENT we have received head data from
// send back a packet with other active node data to this node
foreach (const SharedNodePointer& otherNode, nodeList->getNodeHash()) {
if (otherNode->getLinkedData() && otherNode->getUUID() != node->getUUID()
&& (otherNodeData = reinterpret_cast<AvatarMixerClientData*>(otherNode->getLinkedData()))->getMutex().tryLock()) {
AvatarMixerClientData* otherNodeData = reinterpret_cast<AvatarMixerClientData*>(otherNode->getLinkedData());
AvatarData& otherAvatar = otherNodeData->getAvatar();
glm::vec3 otherPosition = otherAvatar.getPosition();
float distanceToAvatar = glm::length(myPosition - otherPosition);
// The full rate distance is the distance at which EVERY update will be sent for this avatar
// at a distance of twice the full rate distance, there will be a 50% chance of sending this avatar's update
const float FULL_RATE_DISTANCE = 2.0f;
// Decide whether to send this avatar's data based on it's distance from us
if ((_performanceThrottlingRatio == 0 || randFloat() < (1.0f - _performanceThrottlingRatio))
&& (distanceToAvatar == 0.0f || randFloat() < FULL_RATE_DISTANCE / distanceToAvatar)) {
QByteArray avatarByteArray;
avatarByteArray.append(otherNode->getUUID().toRfc4122());
avatarByteArray.append(otherAvatar.toByteArray());
if (avatarByteArray.size() + mixedAvatarByteArray.size() > MAX_PACKET_SIZE) {
nodeList->writeDatagram(mixedAvatarByteArray, node);
// reset the packet
mixedAvatarByteArray.resize(numPacketHeaderBytes);
}
// copy the avatar into the mixedAvatarByteArray packet
mixedAvatarByteArray.append(avatarByteArray);
// if the receiving avatar has just connected make sure we send out the mesh and billboard
// for this avatar (assuming they exist)
bool forceSend = !nodeData->checkAndSetHasReceivedFirstPackets();
// we will also force a send of billboard or identity packet
// if either has changed in the last frame
if (otherNodeData->getBillboardChangeTimestamp() > 0
&& (forceSend
|| otherNodeData->getBillboardChangeTimestamp() > _lastFrameTimestamp
|| randFloat() < BILLBOARD_AND_IDENTITY_SEND_PROBABILITY)) {
QByteArray billboardPacket = byteArrayWithPopulatedHeader(PacketTypeAvatarBillboard);
billboardPacket.append(otherNode->getUUID().toRfc4122());
billboardPacket.append(otherNodeData->getAvatar().getBillboard());
nodeList->writeDatagram(billboardPacket, node);
++_sumBillboardPackets;
}
if (otherNodeData->getIdentityChangeTimestamp() > 0
&& (forceSend
|| otherNodeData->getIdentityChangeTimestamp() > _lastFrameTimestamp
|| randFloat() < BILLBOARD_AND_IDENTITY_SEND_PROBABILITY)) {
QByteArray identityPacket = byteArrayWithPopulatedHeader(PacketTypeAvatarIdentity);
QByteArray individualData = otherNodeData->getAvatar().identityByteArray();
individualData.replace(0, NUM_BYTES_RFC4122_UUID, otherNode->getUUID().toRfc4122());
identityPacket.append(individualData);
nodeList->writeDatagram(identityPacket, node);
++_sumIdentityPackets;
}
}
otherNodeData->getMutex().unlock();
}
}
nodeList->writeDatagram(mixedAvatarByteArray, node);
nodeData->getMutex().unlock();
}
}
void AudioMixer::run() {
commonInit(AUDIO_MIXER_LOGGING_TARGET_NAME, NodeType::AudioMixer);
NodeList* nodeList = NodeList::getInstance();
nodeList->addNodeTypeToInterestSet(NodeType::Agent);
nodeList->linkedDataCreateCallback = attachNewBufferToNode;
int nextFrame = 0;
timeval startTime;
gettimeofday(&startTime, NULL);
char* clientMixBuffer = new char[NETWORK_BUFFER_LENGTH_BYTES_STEREO
+ numBytesForPacketHeaderGivenPacketType(PacketTypeMixedAudio)];
while (!_isFinished) {
foreach (const SharedNodePointer& node, nodeList->getNodeHash()) {
if (node->getLinkedData()) {
((AudioMixerClientData*) node->getLinkedData())->checkBuffersBeforeFrameSend(JITTER_BUFFER_SAMPLES);
}
}
foreach (const SharedNodePointer& node, nodeList->getNodeHash()) {
if (node->getType() == NodeType::Agent && node->getActiveSocket() && node->getLinkedData()
&& ((AudioMixerClientData*) node->getLinkedData())->getAvatarAudioRingBuffer()) {
prepareMixForListeningNode(node.data());
int numBytesPacketHeader = populatePacketHeader(clientMixBuffer, PacketTypeMixedAudio);
memcpy(clientMixBuffer + numBytesPacketHeader, _clientSamples, NETWORK_BUFFER_LENGTH_BYTES_STEREO);
nodeList->writeDatagram(clientMixBuffer, NETWORK_BUFFER_LENGTH_BYTES_STEREO + numBytesPacketHeader, node);
}
}
// push forward the next output pointers for any audio buffers we used
foreach (const SharedNodePointer& node, nodeList->getNodeHash()) {
if (node->getLinkedData()) {
((AudioMixerClientData*) node->getLinkedData())->pushBuffersAfterFrameSend();
}
}
QCoreApplication::processEvents();
if (_isFinished) {
break;
}
int usecToSleep = usecTimestamp(&startTime) + (++nextFrame * BUFFER_SEND_INTERVAL_USECS) - usecTimestampNow();
if (usecToSleep > 0) {
usleep(usecToSleep);
} else {
qDebug() << "AudioMixer loop took" << -usecToSleep << "of extra time. Not sleeping.";
}
}
delete[] clientMixBuffer;
}
void ScriptEngine::run() {
if (!_isInitialized) {
init();
}
_isRunning = true;
_isFinished = false;
emit runningStateChanged();
QScriptValue result = evaluate(_scriptContents);
if (hasUncaughtException()) {
int line = uncaughtExceptionLineNumber();
qDebug() << "Uncaught exception at (" << _fileNameString << ") line" << line << ":" << result.toString();
emit errorMessage("Uncaught exception at (" + _fileNameString + ") line" + QString::number(line) + ":" + result.toString());
clearExceptions();
}
QElapsedTimer startTime;
startTime.start();
int thisFrame = 0;
NodeList* nodeList = NodeList::getInstance();
qint64 lastUpdate = usecTimestampNow();
while (!_isFinished) {
int usecToSleep = (thisFrame++ * SCRIPT_DATA_CALLBACK_USECS) - startTime.nsecsElapsed() / 1000; // nsec to usec
if (usecToSleep > 0) {
usleep(usecToSleep);
}
if (_isFinished) {
break;
}
QCoreApplication::processEvents();
if (_isFinished) {
break;
}
if (_voxelsScriptingInterface.getVoxelPacketSender()->serversExist()) {
// release the queue of edit voxel messages.
_voxelsScriptingInterface.getVoxelPacketSender()->releaseQueuedMessages();
// since we're in non-threaded mode, call process so that the packets are sent
if (!_voxelsScriptingInterface.getVoxelPacketSender()->isThreaded()) {
_voxelsScriptingInterface.getVoxelPacketSender()->process();
}
}
if (_particlesScriptingInterface.getParticlePacketSender()->serversExist()) {
// release the queue of edit voxel messages.
_particlesScriptingInterface.getParticlePacketSender()->releaseQueuedMessages();
// since we're in non-threaded mode, call process so that the packets are sent
if (!_particlesScriptingInterface.getParticlePacketSender()->isThreaded()) {
_particlesScriptingInterface.getParticlePacketSender()->process();
}
}
if (_entityScriptingInterface.getEntityPacketSender()->serversExist()) {
// release the queue of edit voxel messages.
_entityScriptingInterface.getEntityPacketSender()->releaseQueuedMessages();
// since we're in non-threaded mode, call process so that the packets are sent
if (!_entityScriptingInterface.getEntityPacketSender()->isThreaded()) {
_entityScriptingInterface.getEntityPacketSender()->process();
}
}
if (_isAvatar && _avatarData) {
const int SCRIPT_AUDIO_BUFFER_SAMPLES = floor(((SCRIPT_DATA_CALLBACK_USECS * SAMPLE_RATE) / (1000 * 1000)) + 0.5);
const int SCRIPT_AUDIO_BUFFER_BYTES = SCRIPT_AUDIO_BUFFER_SAMPLES * sizeof(int16_t);
QByteArray avatarPacket = byteArrayWithPopulatedHeader(PacketTypeAvatarData);
avatarPacket.append(_avatarData->toByteArray());
nodeList->broadcastToNodes(avatarPacket, NodeSet() << NodeType::AvatarMixer);
if (_isListeningToAudioStream || _avatarSound) {
// if we have an avatar audio stream then send it out to our audio-mixer
bool silentFrame = true;
int16_t numAvailableSamples = SCRIPT_AUDIO_BUFFER_SAMPLES;
const int16_t* nextSoundOutput = NULL;
if (_avatarSound) {
const QByteArray& soundByteArray = _avatarSound->getByteArray();
nextSoundOutput = reinterpret_cast<const int16_t*>(soundByteArray.data()
+ _numAvatarSoundSentBytes);
int numAvailableBytes = (soundByteArray.size() - _numAvatarSoundSentBytes) > SCRIPT_AUDIO_BUFFER_BYTES
? SCRIPT_AUDIO_BUFFER_BYTES
: soundByteArray.size() - _numAvatarSoundSentBytes;
numAvailableSamples = numAvailableBytes / sizeof(int16_t);
// check if the all of the _numAvatarAudioBufferSamples to be sent are silence
for (int i = 0; i < numAvailableSamples; ++i) {
if (nextSoundOutput[i] != 0) {
silentFrame = false;
break;
}
}
_numAvatarSoundSentBytes += numAvailableBytes;
if (_numAvatarSoundSentBytes == soundByteArray.size()) {
// we're done with this sound object - so set our pointer back to NULL
// and our sent bytes back to zero
_avatarSound = NULL;
_numAvatarSoundSentBytes = 0;
}
}
QByteArray audioPacket = byteArrayWithPopulatedHeader(silentFrame
? PacketTypeSilentAudioFrame
: PacketTypeMicrophoneAudioNoEcho);
QDataStream packetStream(&audioPacket, QIODevice::Append);
// pack a placeholder value for sequence number for now, will be packed when destination node is known
int numPreSequenceNumberBytes = audioPacket.size();
packetStream << (quint16) 0;
if (silentFrame) {
if (!_isListeningToAudioStream) {
// if we have a silent frame and we're not listening then just send nothing and break out of here
break;
}
// write the number of silent samples so the audio-mixer can uphold timing
packetStream.writeRawData(reinterpret_cast<const char*>(&SCRIPT_AUDIO_BUFFER_SAMPLES), sizeof(int16_t));
// use the orientation and position of this avatar for the source of this audio
packetStream.writeRawData(reinterpret_cast<const char*>(&_avatarData->getPosition()), sizeof(glm::vec3));
glm::quat headOrientation = _avatarData->getHeadOrientation();
packetStream.writeRawData(reinterpret_cast<const char*>(&headOrientation), sizeof(glm::quat));
} else if (nextSoundOutput) {
// assume scripted avatar audio is mono and set channel flag to zero
packetStream << (quint8)0;
// use the orientation and position of this avatar for the source of this audio
packetStream.writeRawData(reinterpret_cast<const char*>(&_avatarData->getPosition()), sizeof(glm::vec3));
glm::quat headOrientation = _avatarData->getHeadOrientation();
packetStream.writeRawData(reinterpret_cast<const char*>(&headOrientation), sizeof(glm::quat));
// write the raw audio data
packetStream.writeRawData(reinterpret_cast<const char*>(nextSoundOutput), numAvailableSamples * sizeof(int16_t));
}
// write audio packet to AudioMixer nodes
NodeList* nodeList = NodeList::getInstance();
foreach(const SharedNodePointer& node, nodeList->getNodeHash()) {
// only send to nodes of type AudioMixer
if (node->getType() == NodeType::AudioMixer) {
// pack sequence number
quint16 sequence = _outgoingScriptAudioSequenceNumbers[node->getUUID()]++;
memcpy(audioPacket.data() + numPreSequenceNumberBytes, &sequence, sizeof(quint16));
// send audio packet
nodeList->writeDatagram(audioPacket, node);
}
}
}
}
qint64 now = usecTimestampNow();
float deltaTime = (float) (now - lastUpdate) / (float) USECS_PER_SECOND;
if (hasUncaughtException()) {
int line = uncaughtExceptionLineNumber();
qDebug() << "Uncaught exception at (" << _fileNameString << ") line" << line << ":" << uncaughtException().toString();
emit errorMessage("Uncaught exception at (" + _fileNameString + ") line" + QString::number(line) + ":" + uncaughtException().toString());
clearExceptions();
}
emit update(deltaTime);
lastUpdate = now;
}