qint64 NodeList::sendStatsToDomainServer(const QJsonObject& statsObject) {
QByteArray statsPacket = byteArrayWithPopulatedHeader(PacketTypeNodeJsonStats);
QDataStream statsPacketStream(&statsPacket, QIODevice::Append);
statsPacketStream << statsObject.toVariantMap();
return writeUnverifiedDatagram(statsPacket, _domainHandler.getSockAddr());
}
void NodeList::handleICEConnectionToDomainServer() {
if (_domainHandler.getICEPeer().isNull()
|| _domainHandler.getICEPeer().getConnectionAttempts() >= MAX_ICE_CONNECTION_ATTEMPTS) {
_domainHandler.getICEPeer().resetConnectionAttemps();
LimitedNodeList::sendHeartbeatToIceServer(_domainHandler.getICEServerSockAddr(),
_domainHandler.getICEClientID(),
_domainHandler.getICEDomainID());
} else {
qDebug() << "Sending ping packets to establish connectivity with domain-server with ID"
<< uuidStringWithoutCurlyBraces(_domainHandler.getICEDomainID());
// send the ping packet to the local and public sockets for this nodfe
QByteArray localPingPacket = constructPingPacket(PingType::Local, false, _domainHandler.getICEClientID());
writeUnverifiedDatagram(localPingPacket, _domainHandler.getICEPeer().getLocalSocket());
QByteArray publicPingPacket = constructPingPacket(PingType::Public, false, _domainHandler.getICEClientID());
writeUnverifiedDatagram(publicPingPacket, _domainHandler.getICEPeer().getPublicSocket());
_domainHandler.getICEPeer().incrementConnectionAttempts();
}
}
qint64 LimitedNodeList::writeDatagram(const QByteArray& datagram,
const SharedNodePointer& destinationNode,
const HifiSockAddr& overridenSockAddr) {
if (destinationNode) {
PacketType packetType = packetTypeForPacket(datagram);
if (NON_VERIFIED_PACKETS.contains(packetType)) {
return writeUnverifiedDatagram(datagram, destinationNode, overridenSockAddr);
}
// if we don't have an overridden address, assume they want to send to the node's active socket
const HifiSockAddr* destinationSockAddr = &overridenSockAddr;
if (overridenSockAddr.isNull()) {
if (destinationNode->getActiveSocket()) {
// use the node's active socket as the destination socket
destinationSockAddr = destinationNode->getActiveSocket();
} else {
// we don't have a socket to send to, return 0
return 0;
}
}
QByteArray datagramCopy = datagram;
// if we're here and the connection secret is null, debug out - this could be a problem
if (destinationNode->getConnectionSecret().isNull()) {
qDebug() << "LimitedNodeList::writeDatagram called for verified datagram with null connection secret for"
<< "destination node" << destinationNode->getUUID() << " - this is either not secure or will cause"
<< "this packet to be unverifiable on the receiving side.";
}
// perform replacement of hash and optionally also sequence number in the header
if (SEQUENCE_NUMBERED_PACKETS.contains(packetType)) {
PacketSequenceNumber sequenceNumber = getNextSequenceNumberForPacket(destinationNode->getUUID(), packetType);
replaceHashAndSequenceNumberInPacket(datagramCopy, destinationNode->getConnectionSecret(),
sequenceNumber, packetType);
} else {
replaceHashInPacket(datagramCopy, destinationNode->getConnectionSecret(), packetType);
}
emit dataSent(destinationNode->getType(), datagram.size());
auto bytesWritten = writeDatagram(datagramCopy, *destinationSockAddr);
// Keep track of per-destination-node bandwidth
destinationNode->recordBytesSent(bytesWritten);
return bytesWritten;
}
// didn't have a destinationNode to send to, return 0
return 0;
}
void NodeList::processNodeData(const HifiSockAddr& senderSockAddr, const QByteArray& packet) {
switch (packetTypeForPacket(packet)) {
case PacketTypeDomainList: {
processDomainServerList(packet);
break;
}
case PacketTypeDomainServerRequireDTLS: {
_domainHandler.parseDTLSRequirementPacket(packet);
break;
}
case PacketTypeIceServerHeartbeatResponse: {
_domainHandler.processICEResponsePacket(packet);
break;
}
case PacketTypePing: {
// send back a reply
SharedNodePointer matchingNode = sendingNodeForPacket(packet);
if (matchingNode) {
matchingNode->setLastHeardMicrostamp(usecTimestampNow());
QByteArray replyPacket = constructPingReplyPacket(packet);
writeDatagram(replyPacket, matchingNode, senderSockAddr);
// If we don't have a symmetric socket for this node and this socket doesn't match
// what we have for public and local then set it as the symmetric.
// This allows a server on a reachable port to communicate with nodes on symmetric NATs
if (matchingNode->getSymmetricSocket().isNull()) {
if (senderSockAddr != matchingNode->getLocalSocket() && senderSockAddr != matchingNode->getPublicSocket()) {
matchingNode->setSymmetricSocket(senderSockAddr);
}
}
}
break;
}
case PacketTypePingReply: {
SharedNodePointer sendingNode = sendingNodeForPacket(packet);
if (sendingNode) {
sendingNode->setLastHeardMicrostamp(usecTimestampNow());
// activate the appropriate socket for this node, if not yet updated
activateSocketFromNodeCommunication(packet, sendingNode);
// set the ping time for this node for stat collection
timePingReply(packet, sendingNode);
}
break;
}
case PacketTypeUnverifiedPing: {
// send back a reply
QByteArray replyPacket = constructPingReplyPacket(packet, _domainHandler.getICEClientID());
writeUnverifiedDatagram(replyPacket, senderSockAddr);
break;
}
case PacketTypeUnverifiedPingReply: {
qDebug() << "Received reply from domain-server on" << senderSockAddr;
// for now we're unsafely assuming this came back from the domain
if (senderSockAddr == _domainHandler.getICEPeer().getLocalSocket()) {
qDebug() << "Connecting to domain using local socket";
_domainHandler.activateICELocalSocket();
} else if (senderSockAddr == _domainHandler.getICEPeer().getPublicSocket()) {
qDebug() << "Conecting to domain using public socket";
_domainHandler.activateICEPublicSocket();
} else {
qDebug() << "Reply does not match either local or public socket for domain. Will not connect.";
}
}
case PacketTypeStunResponse: {
// a STUN packet begins with 00, we've checked the second zero with packetVersionMatch
// pass it along so it can be processed into our public address and port
processSTUNResponse(packet);
break;
}
default:
LimitedNodeList::processNodeData(senderSockAddr, packet);
break;
}
}
qint64 LimitedNodeList::writeUnverifiedDatagram(const char* data, qint64 size, const SharedNodePointer& destinationNode,
const HifiSockAddr& overridenSockAddr) {
return writeUnverifiedDatagram(QByteArray(data, size), destinationNode, overridenSockAddr);
}
int OctreeInboundPacketProcessor::sendNackPackets() {
int packetsSent = 0;
if (_shuttingDown) {
qDebug() << "OctreeInboundPacketProcessor::sendNackPackets() while shutting down... ignore";
return packetsSent;
}
char packet[MAX_PACKET_SIZE];
NodeToSenderStatsMapIterator i = _singleSenderStats.begin();
while (i != _singleSenderStats.end()) {
QUuid nodeUUID = i.key();
SingleSenderStats nodeStats = i.value();
// check if this node is still alive. Remove its stats if it's dead.
if (!isAlive(nodeUUID)) {
i = _singleSenderStats.erase(i);
continue;
}
// if there are packets from _node that are waiting to be processed,
// don't send a NACK since the missing packets may be among those waiting packets.
if (hasPacketsToProcessFrom(nodeUUID)) {
i++;
continue;
}
const SharedNodePointer& destinationNode = DependencyManager::get<NodeList>()->nodeWithUUID(nodeUUID);
// retrieve sequence number stats of node, prune its missing set
SequenceNumberStats& sequenceNumberStats = nodeStats.getIncomingEditSequenceNumberStats();
sequenceNumberStats.pruneMissingSet();
// construct nack packet(s) for this node
const QSet<unsigned short int>& missingSequenceNumbers = sequenceNumberStats.getMissingSet();
int numSequenceNumbersAvailable = missingSequenceNumbers.size();
QSet<unsigned short int>::const_iterator missingSequenceNumberIterator = missingSequenceNumbers.constBegin();
while (numSequenceNumbersAvailable > 0) {
char* dataAt = packet;
int bytesRemaining = MAX_PACKET_SIZE;
auto nodeList = DependencyManager::get<NodeList>();
// pack header
int numBytesPacketHeader = nodeList->populatePacketHeader(packet, _myServer->getMyEditNackType());
dataAt += numBytesPacketHeader;
bytesRemaining -= numBytesPacketHeader;
// calculate and pack the number of sequence numbers to nack
int numSequenceNumbersRoomFor = (bytesRemaining - sizeof(uint16_t)) / sizeof(unsigned short int);
uint16_t numSequenceNumbers = std::min(numSequenceNumbersAvailable, numSequenceNumbersRoomFor);
uint16_t* numSequenceNumbersAt = (uint16_t*)dataAt;
*numSequenceNumbersAt = numSequenceNumbers;
dataAt += sizeof(uint16_t);
// pack sequence numbers to nack
for (uint16_t i = 0; i < numSequenceNumbers; i++) {
unsigned short int* sequenceNumberAt = (unsigned short int*)dataAt;
*sequenceNumberAt = *missingSequenceNumberIterator;
dataAt += sizeof(unsigned short int);
missingSequenceNumberIterator++;
}
numSequenceNumbersAvailable -= numSequenceNumbers;
// send it
nodeList->writeUnverifiedDatagram(packet, dataAt - packet, destinationNode);
packetsSent++;
qDebug() << "NACK Sent back to editor/client... destinationNode=" << nodeUUID;
}
i++;
}
return packetsSent;
}
void OctreeHeadlessViewer::queryOctree() {
char serverType = getMyNodeType();
PacketType packetType = getMyQueryMessageType();
NodeToJurisdictionMap& jurisdictions = *_jurisdictionListener->getJurisdictions();
bool wantExtraDebugging = false;
if (wantExtraDebugging) {
qCDebug(octree) << "OctreeHeadlessViewer::queryOctree() _jurisdictionListener=" << _jurisdictionListener;
qCDebug(octree) << "---------------";
qCDebug(octree) << "_jurisdictionListener=" << _jurisdictionListener;
qCDebug(octree) << "Jurisdictions...";
jurisdictions.lockForRead();
for (NodeToJurisdictionMapIterator i = jurisdictions.begin(); i != jurisdictions.end(); ++i) {
qCDebug(octree) << i.key() << ": " << &i.value();
}
jurisdictions.unlock();
qCDebug(octree) << "---------------";
}
// These will be the same for all servers, so we can set them up once and then reuse for each server we send to.
_octreeQuery.setWantLowResMoving(true);
_octreeQuery.setWantColor(true);
_octreeQuery.setWantDelta(true);
_octreeQuery.setWantOcclusionCulling(false);
_octreeQuery.setWantCompression(true); // TODO: should be on by default
_octreeQuery.setCameraPosition(_viewFrustum.getPosition());
_octreeQuery.setCameraOrientation(_viewFrustum.getOrientation());
_octreeQuery.setCameraFov(_viewFrustum.getFieldOfView());
_octreeQuery.setCameraAspectRatio(_viewFrustum.getAspectRatio());
_octreeQuery.setCameraNearClip(_viewFrustum.getNearClip());
_octreeQuery.setCameraFarClip(_viewFrustum.getFarClip());
_octreeQuery.setCameraEyeOffsetPosition(_viewFrustum.getEyeOffsetPosition());
_octreeQuery.setOctreeSizeScale(getVoxelSizeScale());
_octreeQuery.setBoundaryLevelAdjust(getBoundaryLevelAdjust());
unsigned char queryPacket[MAX_PACKET_SIZE];
// Iterate all of the nodes, and get a count of how many voxel servers we have...
int totalServers = 0;
int inViewServers = 0;
int unknownJurisdictionServers = 0;
DependencyManager::get<NodeList>()->eachNode([&](const SharedNodePointer& node){
// only send to the NodeTypes that are serverType
if (node->getActiveSocket() && node->getType() == serverType) {
totalServers++;
// get the server bounds for this server
QUuid nodeUUID = node->getUUID();
// if we haven't heard from this voxel server, go ahead and send it a query, so we
// can get the jurisdiction...
jurisdictions.lockForRead();
if (jurisdictions.find(nodeUUID) == jurisdictions.end()) {
jurisdictions.unlock();
unknownJurisdictionServers++;
} else {
const JurisdictionMap& map = (jurisdictions)[nodeUUID];
unsigned char* rootCode = map.getRootOctalCode();
if (rootCode) {
VoxelPositionSize rootDetails;
voxelDetailsForCode(rootCode, rootDetails);
jurisdictions.unlock();
AACube serverBounds(glm::vec3(rootDetails.x, rootDetails.y, rootDetails.z), rootDetails.s);
ViewFrustum::location serverFrustumLocation = _viewFrustum.cubeInFrustum(serverBounds);
if (serverFrustumLocation != ViewFrustum::OUTSIDE) {
inViewServers++;
}
} else {
jurisdictions.unlock();
}
}
}
});
if (wantExtraDebugging) {
qCDebug(octree, "Servers: total %d, in view %d, unknown jurisdiction %d",
totalServers, inViewServers, unknownJurisdictionServers);
}
int perServerPPS = 0;
const int SMALL_BUDGET = 10;
int perUnknownServer = SMALL_BUDGET;
int totalPPS = getMaxPacketsPerSecond();
// determine PPS based on number of servers
if (inViewServers >= 1) {
// set our preferred PPS to be exactly evenly divided among all of the voxel servers... and allocate 1 PPS
// for each unknown jurisdiction server
perServerPPS = (totalPPS / inViewServers) - (unknownJurisdictionServers * perUnknownServer);
} else {
if (unknownJurisdictionServers > 0) {
perUnknownServer = (totalPPS / unknownJurisdictionServers);
}
}
if (wantExtraDebugging) {
qCDebug(octree, "perServerPPS: %d perUnknownServer: %d", perServerPPS, perUnknownServer);
}
auto nodeList = DependencyManager::get<NodeList>();
nodeList->eachNode([&](const SharedNodePointer& node){
// only send to the NodeTypes that are serverType
if (node->getActiveSocket() && node->getType() == serverType) {
// get the server bounds for this server
QUuid nodeUUID = node->getUUID();
bool inView = false;
bool unknownView = false;
// if we haven't heard from this voxel server, go ahead and send it a query, so we
// can get the jurisdiction...
jurisdictions.lockForRead();
if (jurisdictions.find(nodeUUID) == jurisdictions.end()) {
jurisdictions.unlock();
unknownView = true; // assume it's in view
if (wantExtraDebugging) {
qCDebug(octree) << "no known jurisdiction for node " << *node << ", assume it's visible.";
}
} else {
const JurisdictionMap& map = (jurisdictions)[nodeUUID];
unsigned char* rootCode = map.getRootOctalCode();
if (rootCode) {
VoxelPositionSize rootDetails;
voxelDetailsForCode(rootCode, rootDetails);
jurisdictions.unlock();
AACube serverBounds(glm::vec3(rootDetails.x, rootDetails.y, rootDetails.z), rootDetails.s);
ViewFrustum::location serverFrustumLocation = _viewFrustum.cubeInFrustum(serverBounds);
if (serverFrustumLocation != ViewFrustum::OUTSIDE) {
inView = true;
} else {
inView = false;
}
} else {
jurisdictions.unlock();
if (wantExtraDebugging) {
qCDebug(octree) << "Jurisdiction without RootCode for node " << *node << ". That's unusual!";
}
}
}
if (inView) {
_octreeQuery.setMaxQueryPacketsPerSecond(perServerPPS);
if (wantExtraDebugging) {
qCDebug(octree) << "inView for node " << *node << ", give it budget of " << perServerPPS;
}
} else if (unknownView) {
if (wantExtraDebugging) {
qCDebug(octree) << "no known jurisdiction for node " << *node << ", give it budget of "
<< perUnknownServer << " to send us jurisdiction.";
}
// set the query's position/orientation to be degenerate in a manner that will get the scene quickly
// If there's only one server, then don't do this, and just let the normal voxel query pass through
// as expected... this way, we will actually get a valid scene if there is one to be seen
if (totalServers > 1) {
_octreeQuery.setCameraPosition(glm::vec3(-0.1,-0.1,-0.1));
const glm::quat OFF_IN_NEGATIVE_SPACE = glm::quat(-0.5, 0, -0.5, 1.0);
_octreeQuery.setCameraOrientation(OFF_IN_NEGATIVE_SPACE);
_octreeQuery.setCameraNearClip(0.1f);
_octreeQuery.setCameraFarClip(0.1f);
if (wantExtraDebugging) {
qCDebug(octree) << "Using 'minimal' camera position for node" << *node;
}
} else {
if (wantExtraDebugging) {
qCDebug(octree) << "Using regular camera position for node" << *node;
}
}
_octreeQuery.setMaxQueryPacketsPerSecond(perUnknownServer);
} else {
_octreeQuery.setMaxQueryPacketsPerSecond(0);
}
// set up the packet for sending...
unsigned char* endOfQueryPacket = queryPacket;
// insert packet type/version and node UUID
endOfQueryPacket += nodeList->populatePacketHeader(reinterpret_cast<char*>(endOfQueryPacket), packetType);
// encode the query data...
endOfQueryPacket += _octreeQuery.getBroadcastData(endOfQueryPacket);
int packetLength = endOfQueryPacket - queryPacket;
// make sure we still have an active socket
nodeList->writeUnverifiedDatagram(reinterpret_cast<const char*>(queryPacket), packetLength, node);
}
});
}