qint64 NodeList::sendStatsToDomainServer(const QJsonObject& statsObject) {
QByteArray statsPacket = byteArrayWithPopulatedHeader(PacketTypeNodeJsonStats);
QDataStream statsPacketStream(&statsPacket, QIODevice::Append);
statsPacketStream << statsObject.toVariantMap();
return writeUnverifiedDatagram(statsPacket, _domainHandler.getSockAddr());
}
QByteArray LimitedNodeList::constructPingPacket(PingType_t pingType, bool isVerified, const QUuid& packetHeaderID) {
QByteArray pingPacket = byteArrayWithPopulatedHeader(isVerified ? PacketTypePing : PacketTypeUnverifiedPing,
packetHeaderID);
QDataStream packetStream(&pingPacket, QIODevice::Append);
packetStream << pingType;
packetStream << usecTimestampNow();
return pingPacket;
}
void NodeList::sendAssignment(Assignment& assignment) {
PacketType assignmentPacketType = assignment.getCommand() == Assignment::CreateCommand
? PacketTypeCreateAssignment
: PacketTypeRequestAssignment;
QByteArray packet = byteArrayWithPopulatedHeader(assignmentPacketType);
QDataStream packetStream(&packet, QIODevice::Append);
packetStream << assignment;
_nodeSocket.writeDatagram(packet, _assignmentServerSocket.getAddress(), _assignmentServerSocket.getPort());
}
MetavoxelSession::MetavoxelSession(MetavoxelServer* server, const SharedNodePointer& node) :
_server(server),
_sequencer(byteArrayWithPopulatedHeader(PacketTypeMetavoxelData)),
_node(node) {
connect(&_sequencer, SIGNAL(readyToWrite(const QByteArray&)), SLOT(sendData(const QByteArray&)));
connect(&_sequencer, SIGNAL(readyToRead(Bitstream&)), SLOT(readPacket(Bitstream&)));
connect(&_sequencer, SIGNAL(sendAcknowledged(int)), SLOT(clearSendRecordsBefore(int)));
connect(&_sequencer, SIGNAL(receivedHighPriorityMessage(const QVariant&)), SLOT(handleMessage(const QVariant&)));
// insert the baseline send record
SendRecord record = { 0 };
_sendRecords.append(record);
}
Endpoint::Endpoint(const SharedNodePointer& node, PacketRecord* baselineSendRecord, PacketRecord* baselineReceiveRecord) :
_node(node),
_sequencer(byteArrayWithPopulatedHeader(PacketTypeMetavoxelData), this) {
connect(&_sequencer, SIGNAL(readyToWrite(const QByteArray&)), SLOT(sendDatagram(const QByteArray&)));
connect(&_sequencer, SIGNAL(readyToRead(Bitstream&)), SLOT(readMessage(Bitstream&)));
connect(&_sequencer, SIGNAL(sendRecorded()), SLOT(recordSend()));
connect(&_sequencer, SIGNAL(receiveRecorded()), SLOT(recordReceive()));
connect(&_sequencer, SIGNAL(sendAcknowledged(int)), SLOT(clearSendRecordsBefore(int)));
connect(&_sequencer, SIGNAL(receiveAcknowledged(int)), SLOT(clearReceiveRecordsBefore(int)));
// insert the baseline send and receive records
_sendRecords.append(baselineSendRecord);
_receiveRecords.append(baselineReceiveRecord);
}
MetavoxelClient::MetavoxelClient(const SharedNodePointer& node) :
_node(node),
_sequencer(byteArrayWithPopulatedHeader(PacketTypeMetavoxelData)) {
connect(&_sequencer, SIGNAL(readyToWrite(const QByteArray&)), SLOT(sendData(const QByteArray&)));
connect(&_sequencer, SIGNAL(readyToRead(Bitstream&)), SLOT(readPacket(Bitstream&)));
connect(&_sequencer, SIGNAL(sendAcknowledged(int)), SLOT(clearSendRecordsBefore(int)));
connect(&_sequencer, SIGNAL(receiveAcknowledged(int)), SLOT(clearReceiveRecordsBefore(int)));
// insert the baseline send record
SendRecord sendRecord = { 0 };
_sendRecords.append(sendRecord);
// insert the baseline receive record
ReceiveRecord receiveRecord = { 0, _data };
_receiveRecords.append(receiveRecord);
}
void DataServerClient::putValueForKeyAndUserString(const QString& key, const QString& value, const QString& userString) {
// setup the header for this packet and push packetStream to desired spot
QByteArray putPacket = byteArrayWithPopulatedHeader(PacketTypeDataServerPut);
QDataStream packetStream(&putPacket, QIODevice::Append);
// pack our data for the put packet
packetStream << _sequenceNumber << userString << key << value;
// add the putPacket to our vector of unconfirmed packets, will be deleted once put is confirmed
_unmatchedPackets.insert(_sequenceNumber, putPacket);
// send this put request to the data server
NodeList::getInstance()->getNodeSocket().writeDatagram(putPacket, dataServerSockAddr().getAddress(),
dataServerSockAddr().getPort());
// push the sequence number forwards
_sequenceNumber++;
}
void DataServerClient::getValuesForKeysAndUserString(const QStringList& keys, const QString& userString,
DataServerCallbackObject* callbackObject) {
if (!userString.isEmpty() && keys.size() <= UCHAR_MAX) {
QByteArray getPacket = byteArrayWithPopulatedHeader(PacketTypeDataServerGet);
QDataStream packetStream(&getPacket, QIODevice::Append);
// pack our data for the getPacket
packetStream << _sequenceNumber << userString << keys.join(MULTI_KEY_VALUE_SEPARATOR);
// add the getPacket to our map of unconfirmed packets, will be deleted once we get a response from the nameserver
_unmatchedPackets.insert(_sequenceNumber, getPacket);
_callbackObjects.insert(_sequenceNumber, callbackObject);
// send the get to the data server
NodeList::getInstance()->getNodeSocket().writeDatagram(getPacket, dataServerSockAddr().getAddress(),
dataServerSockAddr().getPort());
_sequenceNumber++;
}
}
QByteArray LimitedNodeList::constructPingReplyPacket(const QByteArray& pingPacket, const QUuid& packetHeaderID) {
QDataStream pingPacketStream(pingPacket);
pingPacketStream.skipRawData(numBytesForPacketHeader(pingPacket));
PingType_t typeFromOriginalPing;
pingPacketStream >> typeFromOriginalPing;
quint64 timeFromOriginalPing;
pingPacketStream >> timeFromOriginalPing;
PacketType replyType = (packetTypeForPacket(pingPacket) == PacketTypePing)
? PacketTypePingReply : PacketTypeUnverifiedPingReply;
QByteArray replyPacket = byteArrayWithPopulatedHeader(replyType, packetHeaderID);
QDataStream packetStream(&replyPacket, QIODevice::Append);
packetStream << typeFromOriginalPing << timeFromOriginalPing << usecTimestampNow();
return replyPacket;
}
void AudioInjector::injectAudio() {
QByteArray soundByteArray = _sound->getByteArray();
// make sure we actually have samples downloaded to inject
if (soundByteArray.size()) {
// give our sample byte array to the local audio interface, if we have it, so it can be handled locally
if (_options.getLoopbackAudioInterface()) {
// assume that localAudioInterface could be on a separate thread, use Qt::AutoConnection to handle properly
QMetaObject::invokeMethod(_options.getLoopbackAudioInterface(), "handleAudioByteArray",
Qt::AutoConnection,
Q_ARG(QByteArray, soundByteArray));
}
NodeList* nodeList = NodeList::getInstance();
// setup the packet for injected audio
QByteArray injectAudioPacket = byteArrayWithPopulatedHeader(PacketTypeInjectAudio);
QDataStream packetStream(&injectAudioPacket, QIODevice::Append);
packetStream << QUuid::createUuid();
// pack the flag for loopback
uchar loopbackFlag = (uchar) (!_options.getLoopbackAudioInterface());
packetStream << loopbackFlag;
// pack the position for injected audio
packetStream.writeRawData(reinterpret_cast<const char*>(&_options.getPosition()), sizeof(_options.getPosition()));
// pack our orientation for injected audio
packetStream.writeRawData(reinterpret_cast<const char*>(&_options.getOrientation()), sizeof(_options.getOrientation()));
// pack zero for radius
float radius = 0;
packetStream << radius;
// pack 255 for attenuation byte
quint8 volume = MAX_INJECTOR_VOLUME * _options.getVolume();
packetStream << volume;
QElapsedTimer timer;
timer.start();
int nextFrame = 0;
int currentSendPosition = 0;
int numPreAudioDataBytes = injectAudioPacket.size();
// loop to send off our audio in NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL byte chunks
while (currentSendPosition < soundByteArray.size() && !_shouldStop) {
int bytesToCopy = std::min(NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL,
soundByteArray.size() - currentSendPosition);
// resize the QByteArray to the right size
injectAudioPacket.resize(numPreAudioDataBytes + bytesToCopy);
// copy the next NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL bytes to the packet
memcpy(injectAudioPacket.data() + numPreAudioDataBytes, soundByteArray.data() + currentSendPosition, bytesToCopy);
// grab our audio mixer from the NodeList, if it exists
SharedNodePointer audioMixer = nodeList->soloNodeOfType(NodeType::AudioMixer);
// send off this audio packet
nodeList->writeDatagram(injectAudioPacket, audioMixer);
currentSendPosition += bytesToCopy;
// send two packets before the first sleep so the mixer can start playback right away
if (currentSendPosition != bytesToCopy && currentSendPosition < soundByteArray.size()) {
// not the first packet and not done
// sleep for the appropriate time
int usecToSleep = (++nextFrame * BUFFER_SEND_INTERVAL_USECS) - timer.nsecsElapsed() / 1000;
if (usecToSleep > 0) {
usleep(usecToSleep);
}
}
}
}
emit finished();
}
void NodeList::sendDomainServerCheckIn() {
if (_publicSockAddr.isNull() && !_hasCompletedInitialSTUNFailure) {
// we don't know our public socket and we need to send it to the domain server
// send a STUN request to figure it out
sendSTUNRequest();
} else if (_domainHandler.getIP().isNull() && _domainHandler.requiresICE()) {
handleICEConnectionToDomainServer();
} else if (!_domainHandler.getIP().isNull()) {
bool isUsingDTLS = false;
PacketType domainPacketType = !_domainHandler.isConnected()
? PacketTypeDomainConnectRequest : PacketTypeDomainListRequest;
if (!_domainHandler.isConnected()) {
qDebug() << "Sending connect request to domain-server at" << _domainHandler.getHostname();
}
// construct the DS check in packet
QUuid packetUUID = _sessionUUID;
if (domainPacketType == PacketTypeDomainConnectRequest) {
if (!_domainHandler.getAssignmentUUID().isNull()) {
// this is a connect request and we're an assigned node
// so set our packetUUID as the assignment UUID
packetUUID = _domainHandler.getAssignmentUUID();
} else if (_domainHandler.requiresICE()) {
// this is a connect request and we're an interface client
// that used ice to discover the DS
// so send our ICE client UUID with the connect request
packetUUID = _domainHandler.getICEClientID();
}
}
QByteArray domainServerPacket = byteArrayWithPopulatedHeader(domainPacketType, packetUUID);
QDataStream packetStream(&domainServerPacket, QIODevice::Append);
// pack our data to send to the domain-server
packetStream << _ownerType << _publicSockAddr << _localSockAddr << _nodeTypesOfInterest.toList();
// if this is a connect request, and we can present a username signature, send it along
if (!_domainHandler.isConnected()) {
DataServerAccountInfo& accountInfo = AccountManager::getInstance().getAccountInfo();
packetStream << accountInfo.getUsername();
const QByteArray& usernameSignature = AccountManager::getInstance().getAccountInfo().getUsernameSignature();
if (!usernameSignature.isEmpty()) {
qDebug() << "Including username signature in domain connect request.";
packetStream << usernameSignature;
}
}
if (!isUsingDTLS) {
writeDatagram(domainServerPacket, _domainHandler.getSockAddr(), QUuid());
}
const int NUM_DOMAIN_SERVER_CHECKINS_PER_STUN_REQUEST = 5;
static unsigned int numDomainCheckins = 0;
// send a STUN request every Nth domain server check in so we update our public socket, if required
if (numDomainCheckins++ % NUM_DOMAIN_SERVER_CHECKINS_PER_STUN_REQUEST == 0) {
sendSTUNRequest();
}
if (_numNoReplyDomainCheckIns >= MAX_SILENT_DOMAIN_SERVER_CHECK_INS) {
// we haven't heard back from DS in MAX_SILENT_DOMAIN_SERVER_CHECK_INS
// so emit our signal that says that
emit limitOfSilentDomainCheckInsReached();
}
// increment the count of un-replied check-ins
_numNoReplyDomainCheckIns++;
}
}
void ScriptEngine::run() {
if (!_isInitialized) {
init();
}
_isRunning = true;
emit runningStateChanged();
QScriptValue result = _engine.evaluate(_scriptContents);
if (_engine.hasUncaughtException()) {
int line = _engine.uncaughtExceptionLineNumber();
qDebug() << "Uncaught exception at line" << line << ":" << result.toString();
emit errorMessage("Uncaught exception at line" + QString::number(line) + ":" + result.toString());
}
QElapsedTimer startTime;
startTime.start();
int thisFrame = 0;
NodeList* nodeList = NodeList::getInstance();
qint64 lastUpdate = usecTimestampNow();
while (!_isFinished) {
int usecToSleep = (thisFrame++ * SCRIPT_DATA_CALLBACK_USECS) - startTime.nsecsElapsed() / 1000; // nsec to usec
if (usecToSleep > 0) {
usleep(usecToSleep);
}
if (_isFinished) {
break;
}
QCoreApplication::processEvents();
if (_isFinished) {
break;
}
if (_voxelsScriptingInterface.getVoxelPacketSender()->serversExist()) {
// release the queue of edit voxel messages.
_voxelsScriptingInterface.getVoxelPacketSender()->releaseQueuedMessages();
// since we're in non-threaded mode, call process so that the packets are sent
if (!_voxelsScriptingInterface.getVoxelPacketSender()->isThreaded()) {
_voxelsScriptingInterface.getVoxelPacketSender()->process();
}
}
if (_particlesScriptingInterface.getParticlePacketSender()->serversExist()) {
// release the queue of edit voxel messages.
_particlesScriptingInterface.getParticlePacketSender()->releaseQueuedMessages();
// since we're in non-threaded mode, call process so that the packets are sent
if (!_particlesScriptingInterface.getParticlePacketSender()->isThreaded()) {
_particlesScriptingInterface.getParticlePacketSender()->process();
}
}
if (_modelsScriptingInterface.getModelPacketSender()->serversExist()) {
// release the queue of edit voxel messages.
_modelsScriptingInterface.getModelPacketSender()->releaseQueuedMessages();
// since we're in non-threaded mode, call process so that the packets are sent
if (!_modelsScriptingInterface.getModelPacketSender()->isThreaded()) {
_modelsScriptingInterface.getModelPacketSender()->process();
}
}
if (_isAvatar && _avatarData) {
const int SCRIPT_AUDIO_BUFFER_SAMPLES = floor(((SCRIPT_DATA_CALLBACK_USECS * SAMPLE_RATE) / (1000 * 1000)) + 0.5);
const int SCRIPT_AUDIO_BUFFER_BYTES = SCRIPT_AUDIO_BUFFER_SAMPLES * sizeof(int16_t);
QByteArray avatarPacket = byteArrayWithPopulatedHeader(PacketTypeAvatarData);
avatarPacket.append(_avatarData->toByteArray());
nodeList->broadcastToNodes(avatarPacket, NodeSet() << NodeType::AvatarMixer);
if (_isListeningToAudioStream || _avatarSound) {
// if we have an avatar audio stream then send it out to our audio-mixer
bool silentFrame = true;
int16_t numAvailableSamples = SCRIPT_AUDIO_BUFFER_SAMPLES;
const int16_t* nextSoundOutput = NULL;
if (_avatarSound) {
const QByteArray& soundByteArray = _avatarSound->getByteArray();
nextSoundOutput = reinterpret_cast<const int16_t*>(soundByteArray.data()
+ _numAvatarSoundSentBytes);
int numAvailableBytes = (soundByteArray.size() - _numAvatarSoundSentBytes) > SCRIPT_AUDIO_BUFFER_BYTES
? SCRIPT_AUDIO_BUFFER_BYTES
: soundByteArray.size() - _numAvatarSoundSentBytes;
numAvailableSamples = numAvailableBytes / sizeof(int16_t);
// check if the all of the _numAvatarAudioBufferSamples to be sent are silence
for (int i = 0; i < numAvailableSamples; ++i) {
if (nextSoundOutput[i] != 0) {
silentFrame = false;
break;
}
}
_numAvatarSoundSentBytes += numAvailableBytes;
if (_numAvatarSoundSentBytes == soundByteArray.size()) {
// we're done with this sound object - so set our pointer back to NULL
// and our sent bytes back to zero
_avatarSound = NULL;
_numAvatarSoundSentBytes = 0;
}
}
QByteArray audioPacket = byteArrayWithPopulatedHeader(silentFrame
? PacketTypeSilentAudioFrame
: PacketTypeMicrophoneAudioNoEcho);
QDataStream packetStream(&audioPacket, QIODevice::Append);
// use the orientation and position of this avatar for the source of this audio
packetStream.writeRawData(reinterpret_cast<const char*>(&_avatarData->getPosition()), sizeof(glm::vec3));
glm::quat headOrientation = _avatarData->getHeadOrientation();
packetStream.writeRawData(reinterpret_cast<const char*>(&headOrientation), sizeof(glm::quat));
if (silentFrame) {
if (!_isListeningToAudioStream) {
// if we have a silent frame and we're not listening then just send nothing and break out of here
break;
}
// write the number of silent samples so the audio-mixer can uphold timing
packetStream.writeRawData(reinterpret_cast<const char*>(&SCRIPT_AUDIO_BUFFER_SAMPLES), sizeof(int16_t));
} else if (nextSoundOutput) {
// write the raw audio data
packetStream.writeRawData(reinterpret_cast<const char*>(nextSoundOutput),
numAvailableSamples * sizeof(int16_t));
}
nodeList->broadcastToNodes(audioPacket, NodeSet() << NodeType::AudioMixer);
}
}
qint64 now = usecTimestampNow();
float deltaTime = (float) (now - lastUpdate) / (float) USECS_PER_SECOND;
emit update(deltaTime);
lastUpdate = now;
if (_engine.hasUncaughtException()) {
int line = _engine.uncaughtExceptionLineNumber();
qDebug() << "Uncaught exception at line" << line << ":" << _engine.uncaughtException().toString();
emit errorMessage("Uncaught exception at line" + QString::number(line) + ":" + _engine.uncaughtException().toString());
}
}
emit scriptEnding();
// kill the avatar identity timer
delete _avatarIdentityTimer;
if (_voxelsScriptingInterface.getVoxelPacketSender()->serversExist()) {
// release the queue of edit voxel messages.
_voxelsScriptingInterface.getVoxelPacketSender()->releaseQueuedMessages();
// since we're in non-threaded mode, call process so that the packets are sent
if (!_voxelsScriptingInterface.getVoxelPacketSender()->isThreaded()) {
_voxelsScriptingInterface.getVoxelPacketSender()->process();
}
}
if (_particlesScriptingInterface.getParticlePacketSender()->serversExist()) {
// release the queue of edit voxel messages.
_particlesScriptingInterface.getParticlePacketSender()->releaseQueuedMessages();
// since we're in non-threaded mode, call process so that the packets are sent
if (!_particlesScriptingInterface.getParticlePacketSender()->isThreaded()) {
_particlesScriptingInterface.getParticlePacketSender()->process();
}
}
if (_modelsScriptingInterface.getModelPacketSender()->serversExist()) {
// release the queue of edit voxel messages.
_modelsScriptingInterface.getModelPacketSender()->releaseQueuedMessages();
// since we're in non-threaded mode, call process so that the packets are sent
if (!_modelsScriptingInterface.getModelPacketSender()->isThreaded()) {
_modelsScriptingInterface.getModelPacketSender()->process();
}
}
// If we were on a thread, then wait till it's done
if (thread()) {
thread()->quit();
}
emit finished(_fileNameString);
_isRunning = false;
emit runningStateChanged();
}
void AudioInjector::injectToMixer() {
if (_currentSendPosition < 0 ||
_currentSendPosition >= _audioData.size()) {
_currentSendPosition = 0;
}
auto nodeList = DependencyManager::get<NodeList>();
// make sure we actually have samples downloaded to inject
if (_audioData.size()) {
// setup the packet for injected audio
QByteArray injectAudioPacket = nodeList->byteArrayWithPopulatedHeader(PacketTypeInjectAudio);
QDataStream packetStream(&injectAudioPacket, QIODevice::Append);
// pack some placeholder sequence number for now
int numPreSequenceNumberBytes = injectAudioPacket.size();
packetStream << (quint16)0;
// pack stream identifier (a generated UUID)
packetStream << QUuid::createUuid();
// pack the stereo/mono type of the stream
packetStream << _options.stereo;
// pack the flag for loopback
uchar loopbackFlag = (uchar) true;
packetStream << loopbackFlag;
// pack the position for injected audio
int positionOptionOffset = injectAudioPacket.size();
packetStream.writeRawData(reinterpret_cast<const char*>(&_options.position),
sizeof(_options.position));
// pack our orientation for injected audio
int orientationOptionOffset = injectAudioPacket.size();
packetStream.writeRawData(reinterpret_cast<const char*>(&_options.orientation),
sizeof(_options.orientation));
// pack zero for radius
float radius = 0;
packetStream << radius;
// pack 255 for attenuation byte
int volumeOptionOffset = injectAudioPacket.size();
quint8 volume = MAX_INJECTOR_VOLUME * _options.volume;
packetStream << volume;
packetStream << _options.ignorePenumbra;
QElapsedTimer timer;
timer.start();
int nextFrame = 0;
int numPreAudioDataBytes = injectAudioPacket.size();
bool shouldLoop = _options.loop;
// loop to send off our audio in NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL byte chunks
quint16 outgoingInjectedAudioSequenceNumber = 0;
while (_currentSendPosition < _audioData.size() && !_shouldStop) {
int bytesToCopy = std::min(((_options.stereo) ? 2 : 1) * AudioConstants::NETWORK_FRAME_BYTES_PER_CHANNEL,
_audioData.size() - _currentSendPosition);
// Measure the loudness of this frame
_loudness = 0.0f;
for (int i = 0; i < bytesToCopy; i += sizeof(int16_t)) {
_loudness += abs(*reinterpret_cast<int16_t*>(_audioData.data() + _currentSendPosition + i)) /
(AudioConstants::MAX_SAMPLE_VALUE / 2.0f);
}
_loudness /= (float)(bytesToCopy / sizeof(int16_t));
memcpy(injectAudioPacket.data() + positionOptionOffset,
&_options.position,
sizeof(_options.position));
memcpy(injectAudioPacket.data() + orientationOptionOffset,
&_options.orientation,
sizeof(_options.orientation));
volume = MAX_INJECTOR_VOLUME * _options.volume;
memcpy(injectAudioPacket.data() + volumeOptionOffset, &volume, sizeof(volume));
// resize the QByteArray to the right size
injectAudioPacket.resize(numPreAudioDataBytes + bytesToCopy);
// pack the sequence number
memcpy(injectAudioPacket.data() + numPreSequenceNumberBytes,
&outgoingInjectedAudioSequenceNumber, sizeof(quint16));
// copy the next NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL bytes to the packet
memcpy(injectAudioPacket.data() + numPreAudioDataBytes,
_audioData.data() + _currentSendPosition, bytesToCopy);
// grab our audio mixer from the NodeList, if it exists
SharedNodePointer audioMixer = nodeList->soloNodeOfType(NodeType::AudioMixer);
// send off this audio packet
nodeList->writeDatagram(injectAudioPacket, audioMixer);
outgoingInjectedAudioSequenceNumber++;
_currentSendPosition += bytesToCopy;
// send two packets before the first sleep so the mixer can start playback right away
if (_currentSendPosition != bytesToCopy && _currentSendPosition < _audioData.size()) {
// process events in case we have been told to stop and be deleted
QCoreApplication::processEvents();
if (_shouldStop) {
break;
}
// not the first packet and not done
// sleep for the appropriate time
int usecToSleep = (++nextFrame * AudioConstants::NETWORK_FRAME_USECS) - timer.nsecsElapsed() / 1000;
if (usecToSleep > 0) {
usleep(usecToSleep);
}
}
if (shouldLoop && _currentSendPosition >= _audioData.size()) {
_currentSendPosition = 0;
}
}
}
setIsFinished(true);
}
void ScriptEngine::run() {
// TODO: can we add a short circuit for _stoppingAllScripts here? What does it mean to not start running if
// we're in the process of stopping?
if (!_isInitialized) {
init();
}
_isRunning = true;
_isFinished = false;
emit runningStateChanged();
QScriptValue result = evaluate(_scriptContents);
QElapsedTimer startTime;
startTime.start();
int thisFrame = 0;
auto nodeList = DependencyManager::get<NodeList>();
auto entityScriptingInterface = DependencyManager::get<EntityScriptingInterface>();
qint64 lastUpdate = usecTimestampNow();
while (!_isFinished) {
int usecToSleep = (thisFrame++ * SCRIPT_DATA_CALLBACK_USECS) - startTime.nsecsElapsed() / 1000; // nsec to usec
if (usecToSleep > 0) {
usleep(usecToSleep);
}
if (_isFinished) {
break;
}
QCoreApplication::processEvents();
if (_isFinished) {
break;
}
if (!_isFinished && entityScriptingInterface->getEntityPacketSender()->serversExist()) {
// release the queue of edit entity messages.
entityScriptingInterface->getEntityPacketSender()->releaseQueuedMessages();
// since we're in non-threaded mode, call process so that the packets are sent
if (!entityScriptingInterface->getEntityPacketSender()->isThreaded()) {
entityScriptingInterface->getEntityPacketSender()->process();
}
}
if (!_isFinished && _isAvatar && _avatarData) {
const int SCRIPT_AUDIO_BUFFER_SAMPLES = floor(((SCRIPT_DATA_CALLBACK_USECS * AudioConstants::SAMPLE_RATE)
/ (1000 * 1000)) + 0.5);
const int SCRIPT_AUDIO_BUFFER_BYTES = SCRIPT_AUDIO_BUFFER_SAMPLES * sizeof(int16_t);
QByteArray avatarPacket = byteArrayWithPopulatedHeader(PacketTypeAvatarData);
avatarPacket.append(_avatarData->toByteArray());
nodeList->broadcastToNodes(avatarPacket, NodeSet() << NodeType::AvatarMixer);
if (_isListeningToAudioStream || _avatarSound) {
// if we have an avatar audio stream then send it out to our audio-mixer
bool silentFrame = true;
int16_t numAvailableSamples = SCRIPT_AUDIO_BUFFER_SAMPLES;
const int16_t* nextSoundOutput = NULL;
if (_avatarSound) {
const QByteArray& soundByteArray = _avatarSound->getByteArray();
nextSoundOutput = reinterpret_cast<const int16_t*>(soundByteArray.data()
+ _numAvatarSoundSentBytes);
int numAvailableBytes = (soundByteArray.size() - _numAvatarSoundSentBytes) > SCRIPT_AUDIO_BUFFER_BYTES
? SCRIPT_AUDIO_BUFFER_BYTES
: soundByteArray.size() - _numAvatarSoundSentBytes;
numAvailableSamples = numAvailableBytes / sizeof(int16_t);
// check if the all of the _numAvatarAudioBufferSamples to be sent are silence
for (int i = 0; i < numAvailableSamples; ++i) {
if (nextSoundOutput[i] != 0) {
silentFrame = false;
break;
}
}
_numAvatarSoundSentBytes += numAvailableBytes;
if (_numAvatarSoundSentBytes == soundByteArray.size()) {
// we're done with this sound object - so set our pointer back to NULL
// and our sent bytes back to zero
_avatarSound = NULL;
_numAvatarSoundSentBytes = 0;
}
}
QByteArray audioPacket = byteArrayWithPopulatedHeader(silentFrame
? PacketTypeSilentAudioFrame
: PacketTypeMicrophoneAudioNoEcho);
QDataStream packetStream(&audioPacket, QIODevice::Append);
// pack a placeholder value for sequence number for now, will be packed when destination node is known
int numPreSequenceNumberBytes = audioPacket.size();
packetStream << (quint16) 0;
if (silentFrame) {
if (!_isListeningToAudioStream) {
// if we have a silent frame and we're not listening then just send nothing and break out of here
break;
}
// write the number of silent samples so the audio-mixer can uphold timing
packetStream.writeRawData(reinterpret_cast<const char*>(&SCRIPT_AUDIO_BUFFER_SAMPLES), sizeof(int16_t));
// use the orientation and position of this avatar for the source of this audio
packetStream.writeRawData(reinterpret_cast<const char*>(&_avatarData->getPosition()), sizeof(glm::vec3));
glm::quat headOrientation = _avatarData->getHeadOrientation();
packetStream.writeRawData(reinterpret_cast<const char*>(&headOrientation), sizeof(glm::quat));
} else if (nextSoundOutput) {
// assume scripted avatar audio is mono and set channel flag to zero
packetStream << (quint8)0;
// use the orientation and position of this avatar for the source of this audio
packetStream.writeRawData(reinterpret_cast<const char*>(&_avatarData->getPosition()), sizeof(glm::vec3));
glm::quat headOrientation = _avatarData->getHeadOrientation();
packetStream.writeRawData(reinterpret_cast<const char*>(&headOrientation), sizeof(glm::quat));
// write the raw audio data
packetStream.writeRawData(reinterpret_cast<const char*>(nextSoundOutput), numAvailableSamples * sizeof(int16_t));
}
// write audio packet to AudioMixer nodes
auto nodeList = DependencyManager::get<NodeList>();
nodeList->eachNode([this, &nodeList, &audioPacket, &numPreSequenceNumberBytes](const SharedNodePointer& node){
// only send to nodes of type AudioMixer
if (node->getType() == NodeType::AudioMixer) {
// pack sequence number
quint16 sequence = _outgoingScriptAudioSequenceNumbers[node->getUUID()]++;
memcpy(audioPacket.data() + numPreSequenceNumberBytes, &sequence, sizeof(quint16));
// send audio packet
nodeList->writeDatagram(audioPacket, node);
}
});
}
}
qint64 now = usecTimestampNow();
float deltaTime = (float) (now - lastUpdate) / (float) USECS_PER_SECOND;
if (hasUncaughtException()) {
int line = uncaughtExceptionLineNumber();
qCDebug(scriptengine) << "Uncaught exception at (" << _fileNameString << ") line" << line << ":" << uncaughtException().toString();
emit errorMessage("Uncaught exception at (" + _fileNameString + ") line" + QString::number(line) + ":" + uncaughtException().toString());
clearExceptions();
}
if (!_isFinished) {
emit update(deltaTime);
}
lastUpdate = now;
}
stopAllTimers(); // make sure all our timers are stopped if the script is ending
emit scriptEnding();
// kill the avatar identity timer
delete _avatarIdentityTimer;
if (entityScriptingInterface->getEntityPacketSender()->serversExist()) {
// release the queue of edit entity messages.
entityScriptingInterface->getEntityPacketSender()->releaseQueuedMessages();
// since we're in non-threaded mode, call process so that the packets are sent
if (!entityScriptingInterface->getEntityPacketSender()->isThreaded()) {
entityScriptingInterface->getEntityPacketSender()->process();
}
}
// If we were on a thread, then wait till it's done
if (thread()) {
thread()->quit();
}
emit finished(_fileNameString);
_isRunning = false;
emit runningStateChanged();
emit doneRunning();
_doneRunningThisScript = true;
}
// NOTE: some additional optimizations to consider.
// 1) use the view frustum to cull those avatars that are out of view. Since avatar data doesn't need to be present
// if the avatar is not in view or in the keyhole.
void AvatarMixer::broadcastAvatarData() {
int idleTime = QDateTime::currentMSecsSinceEpoch() - _lastFrameTimestamp;
++_numStatFrames;
const float STRUGGLE_TRIGGER_SLEEP_PERCENTAGE_THRESHOLD = 0.10f;
const float BACK_OFF_TRIGGER_SLEEP_PERCENTAGE_THRESHOLD = 0.20f;
const float RATIO_BACK_OFF = 0.02f;
const int TRAILING_AVERAGE_FRAMES = 100;
int framesSinceCutoffEvent = TRAILING_AVERAGE_FRAMES;
const float CURRENT_FRAME_RATIO = 1.0f / TRAILING_AVERAGE_FRAMES;
const float PREVIOUS_FRAMES_RATIO = 1.0f - CURRENT_FRAME_RATIO;
_trailingSleepRatio = (PREVIOUS_FRAMES_RATIO * _trailingSleepRatio)
+ (idleTime * CURRENT_FRAME_RATIO / (float) AVATAR_DATA_SEND_INTERVAL_MSECS);
float lastCutoffRatio = _performanceThrottlingRatio;
bool hasRatioChanged = false;
if (framesSinceCutoffEvent >= TRAILING_AVERAGE_FRAMES) {
if (_trailingSleepRatio <= STRUGGLE_TRIGGER_SLEEP_PERCENTAGE_THRESHOLD) {
// we're struggling - change our min required loudness to reduce some load
_performanceThrottlingRatio = _performanceThrottlingRatio + (0.5f * (1.0f - _performanceThrottlingRatio));
qDebug() << "Mixer is struggling, sleeping" << _trailingSleepRatio * 100 << "% of frame time. Old cutoff was"
<< lastCutoffRatio << "and is now" << _performanceThrottlingRatio;
hasRatioChanged = true;
} else if (_trailingSleepRatio >= BACK_OFF_TRIGGER_SLEEP_PERCENTAGE_THRESHOLD && _performanceThrottlingRatio != 0) {
// we've recovered and can back off the required loudness
_performanceThrottlingRatio = _performanceThrottlingRatio - RATIO_BACK_OFF;
if (_performanceThrottlingRatio < 0) {
_performanceThrottlingRatio = 0;
}
qDebug() << "Mixer is recovering, sleeping" << _trailingSleepRatio * 100 << "% of frame time. Old cutoff was"
<< lastCutoffRatio << "and is now" << _performanceThrottlingRatio;
hasRatioChanged = true;
}
if (hasRatioChanged) {
framesSinceCutoffEvent = 0;
}
}
if (!hasRatioChanged) {
++framesSinceCutoffEvent;
}
static QByteArray mixedAvatarByteArray;
int numPacketHeaderBytes = populatePacketHeader(mixedAvatarByteArray, PacketTypeBulkAvatarData);
NodeList* nodeList = NodeList::getInstance();
AvatarMixerClientData* nodeData = NULL;
AvatarMixerClientData* otherNodeData = NULL;
foreach (const SharedNodePointer& node, nodeList->getNodeHash()) {
if (node->getLinkedData() && node->getType() == NodeType::Agent && node->getActiveSocket()
&& (nodeData = reinterpret_cast<AvatarMixerClientData*>(node->getLinkedData()))->getMutex().tryLock()) {
++_sumListeners;
// reset packet pointers for this node
mixedAvatarByteArray.resize(numPacketHeaderBytes);
AvatarData& avatar = nodeData->getAvatar();
glm::vec3 myPosition = avatar.getPosition();
// this is an AGENT we have received head data from
// send back a packet with other active node data to this node
foreach (const SharedNodePointer& otherNode, nodeList->getNodeHash()) {
if (otherNode->getLinkedData() && otherNode->getUUID() != node->getUUID()
&& (otherNodeData = reinterpret_cast<AvatarMixerClientData*>(otherNode->getLinkedData()))->getMutex().tryLock()) {
AvatarMixerClientData* otherNodeData = reinterpret_cast<AvatarMixerClientData*>(otherNode->getLinkedData());
AvatarData& otherAvatar = otherNodeData->getAvatar();
glm::vec3 otherPosition = otherAvatar.getPosition();
float distanceToAvatar = glm::length(myPosition - otherPosition);
// The full rate distance is the distance at which EVERY update will be sent for this avatar
// at a distance of twice the full rate distance, there will be a 50% chance of sending this avatar's update
const float FULL_RATE_DISTANCE = 2.0f;
// Decide whether to send this avatar's data based on it's distance from us
if ((_performanceThrottlingRatio == 0 || randFloat() < (1.0f - _performanceThrottlingRatio))
&& (distanceToAvatar == 0.0f || randFloat() < FULL_RATE_DISTANCE / distanceToAvatar)) {
QByteArray avatarByteArray;
avatarByteArray.append(otherNode->getUUID().toRfc4122());
avatarByteArray.append(otherAvatar.toByteArray());
if (avatarByteArray.size() + mixedAvatarByteArray.size() > MAX_PACKET_SIZE) {
nodeList->writeDatagram(mixedAvatarByteArray, node);
// reset the packet
mixedAvatarByteArray.resize(numPacketHeaderBytes);
}
// copy the avatar into the mixedAvatarByteArray packet
mixedAvatarByteArray.append(avatarByteArray);
// if the receiving avatar has just connected make sure we send out the mesh and billboard
// for this avatar (assuming they exist)
bool forceSend = !nodeData->checkAndSetHasReceivedFirstPackets();
// we will also force a send of billboard or identity packet
// if either has changed in the last frame
if (otherNodeData->getBillboardChangeTimestamp() > 0
&& (forceSend
|| otherNodeData->getBillboardChangeTimestamp() > _lastFrameTimestamp
|| randFloat() < BILLBOARD_AND_IDENTITY_SEND_PROBABILITY)) {
QByteArray billboardPacket = byteArrayWithPopulatedHeader(PacketTypeAvatarBillboard);
billboardPacket.append(otherNode->getUUID().toRfc4122());
billboardPacket.append(otherNodeData->getAvatar().getBillboard());
nodeList->writeDatagram(billboardPacket, node);
++_sumBillboardPackets;
}
if (otherNodeData->getIdentityChangeTimestamp() > 0
&& (forceSend
|| otherNodeData->getIdentityChangeTimestamp() > _lastFrameTimestamp
|| randFloat() < BILLBOARD_AND_IDENTITY_SEND_PROBABILITY)) {
QByteArray identityPacket = byteArrayWithPopulatedHeader(PacketTypeAvatarIdentity);
QByteArray individualData = otherNodeData->getAvatar().identityByteArray();
individualData.replace(0, NUM_BYTES_RFC4122_UUID, otherNode->getUUID().toRfc4122());
identityPacket.append(individualData);
nodeList->writeDatagram(identityPacket, node);
++_sumIdentityPackets;
}
}
otherNodeData->getMutex().unlock();
}
}
nodeList->writeDatagram(mixedAvatarByteArray, node);
nodeData->getMutex().unlock();
}
}