void AgentList::timePingReply(sockaddr *agentAddress, unsigned char *packetData) { for(AgentList::iterator agent = begin(); agent != end(); agent++) { if (socketMatch(agent->getPublicSocket(), agentAddress) || socketMatch(agent->getLocalSocket(), agentAddress)) { int pingTime = usecTimestampNow() - *(long long *)(packetData + 1); agent->setPingMs(pingTime / 1000); break; } } }
void NodeList::timePingReply(sockaddr *nodeAddress, unsigned char *packetData) { for(NodeList::iterator node = begin(); node != end(); node++) { if (socketMatch(node->getPublicSocket(), nodeAddress) || socketMatch(node->getLocalSocket(), nodeAddress)) { int pingTime = usecTimestampNow() - *(uint64_t*)(packetData + numBytesForPacketHeader(packetData)); node->setPingMs(pingTime / 1000); break; } } }
void AgentList::handlePingReply(sockaddr *agentAddress) { for(AgentList::iterator agent = begin(); agent != end(); agent++) { // check both the public and local addresses for each agent to see if we find a match // prioritize the private address so that we prune erroneous local matches if (socketMatch(agent->getPublicSocket(), agentAddress)) { agent->activatePublicSocket(); break; } else if (socketMatch(agent->getLocalSocket(), agentAddress)) { agent->activateLocalSocket(); break; } } }
void NodeList::handlePingReply(sockaddr *nodeAddress) { for(NodeList::iterator node = begin(); node != end(); node++) { // check both the public and local addresses for each node to see if we find a match // prioritize the private address so that we prune erroneous local matches if (socketMatch(node->getPublicSocket(), nodeAddress)) { node->activatePublicSocket(); break; } else if (socketMatch(node->getLocalSocket(), nodeAddress)) { node->activateLocalSocket(); break; } } }
bool DomainServer::checkInWithUUIDMatchesExistingNode(sockaddr* nodePublicSocket, sockaddr* nodeLocalSocket, const QUuid& checkInUUID) { NodeList* nodeList = NodeList::getInstance(); for (NodeList::iterator node = nodeList->begin(); node != nodeList->end(); node++) { if (node->getLinkedData() && socketMatch(node->getPublicSocket(), nodePublicSocket) && socketMatch(node->getLocalSocket(), nodeLocalSocket) && node->getUUID() == checkInUUID) { // this is a matching existing node if the public socket, local socket, and UUID match return true; } } return false; }
Agent* AgentList::agentWithAddress(sockaddr *senderAddress) { for(AgentList::iterator agent = begin(); agent != end(); agent++) { if (agent->getActiveSocket() && socketMatch(agent->getActiveSocket(), senderAddress)) { return &(*agent); } } return NULL; }
Node* NodeList::nodeWithAddress(sockaddr *senderAddress) { for(NodeList::iterator node = begin(); node != end(); node++) { if (node->getActiveSocket() && socketMatch(node->getActiveSocket(), senderAddress)) { return &(*node); } } return NULL; }
bool DomainServer::checkInWithUUIDMatchesExistingNode(sockaddr* nodePublicSocket, sockaddr* nodeLocalSocket, const uchar* checkInData) { // pull the UUID passed with the check in QUuid checkInUUID = QUuid::fromRfc4122(QByteArray((const char*) checkInData + numBytesForPacketHeader(checkInData) + sizeof(NODE_TYPE), NUM_BYTES_RFC4122_UUID)); NodeList* nodeList = NodeList::getInstance(); for (NodeList::iterator node = nodeList->begin(); node != nodeList->end(); node++) { if (node->getLinkedData() && socketMatch(node->getPublicSocket(), nodePublicSocket) && socketMatch(node->getLocalSocket(), nodeLocalSocket) && ((Assignment*) node->getLinkedData())->getUUID() == checkInUUID) { // this is a matching existing node if the public socket, local socket, and UUID match return true; } } return false; }
Node* NodeList::addOrUpdateNode(sockaddr* publicSocket, sockaddr* localSocket, char nodeType, uint16_t nodeId) { NodeList::iterator node = end(); if (publicSocket) { for (node = begin(); node != end(); node++) { if (node->matches(publicSocket, localSocket, nodeType)) { // we already have this node, stop checking break; } } } if (node == end()) { // if we already had this node AND it's a solo type then bust out of here if (soloNodeOfType(nodeType)) { return NULL; } // we didn't have this node, so add them Node* newNode = new Node(publicSocket, localSocket, nodeType, nodeId); if (socketMatch(publicSocket, localSocket)) { // likely debugging scenario with two nodes on local network // set the node active right away newNode->activatePublicSocket(); } if (newNode->getType() == NODE_TYPE_VOXEL_SERVER || newNode->getType() == NODE_TYPE_AVATAR_MIXER || newNode->getType() == NODE_TYPE_AUDIO_MIXER) { // this is currently the cheat we use to talk directly to our test servers on EC2 // to be removed when we have a proper identification strategy newNode->activatePublicSocket(); } addNodeToList(newNode); return newNode; } else { if (node->getType() == NODE_TYPE_AUDIO_MIXER || node->getType() == NODE_TYPE_VOXEL_SERVER) { // until the Audio class also uses our nodeList, we need to update // the lastRecvTimeUsecs for the audio mixer so it doesn't get killed and re-added continously node->setLastHeardMicrostamp(usecTimestampNow()); } // we had this node already, do nothing for now return &*node; } }
Agent* AgentList::addOrUpdateAgent(sockaddr* publicSocket, sockaddr* localSocket, char agentType, uint16_t agentId) { AgentList::iterator agent = end(); if (publicSocket) { for (agent = begin(); agent != end(); agent++) { if (agent->matches(publicSocket, localSocket, agentType)) { // we already have this agent, stop checking break; } } } if (agent == end()) { // we didn't have this agent, so add them Agent* newAgent = new Agent(publicSocket, localSocket, agentType, agentId); if (socketMatch(publicSocket, localSocket)) { // likely debugging scenario with two agents on local network // set the agent active right away newAgent->activatePublicSocket(); } if (newAgent->getType() == AGENT_TYPE_VOXEL_SERVER || newAgent->getType() == AGENT_TYPE_AVATAR_MIXER || newAgent->getType() == AGENT_TYPE_AUDIO_MIXER) { // this is currently the cheat we use to talk directly to our test servers on EC2 // to be removed when we have a proper identification strategy newAgent->activatePublicSocket(); } addAgentToList(newAgent); return newAgent; } else { if (agent->getType() == AGENT_TYPE_AUDIO_MIXER || agent->getType() == AGENT_TYPE_VOXEL_SERVER) { // until the Audio class also uses our agentList, we need to update // the lastRecvTimeUsecs for the audio mixer so it doesn't get killed and re-added continously agent->setLastHeardMicrostamp(usecTimestampNow()); } // we had this agent already, do nothing for now return &*agent; } }
// NOTE: some additional optimizations to consider. // 1) use the view frustum to cull those avatars that are out of view. Since avatar data doesn't need to be present // if the avatar is not in view or in the keyhole. // 2) after culling for view frustum, sort order the avatars by distance, send the closest ones first. // 3) if we need to rate limit the amount of data we send, we can use a distance weighted "semi-random" function to // determine which avatars are included in the packet stream // 4) we should optimize the avatar data format to be more compact (100 bytes is pretty wasteful). void broadcastAvatarData(NodeList* nodeList, sockaddr* nodeAddress) { static unsigned char broadcastPacketBuffer[MAX_PACKET_SIZE]; static unsigned char avatarDataBuffer[MAX_PACKET_SIZE]; unsigned char* broadcastPacket = (unsigned char*)&broadcastPacketBuffer[0]; int numHeaderBytes = populateTypeAndVersion(broadcastPacket, PACKET_TYPE_BULK_AVATAR_DATA); unsigned char* currentBufferPosition = broadcastPacket + numHeaderBytes; int packetLength = currentBufferPosition - broadcastPacket; int packetsSent = 0; // send back a packet with other active node data to this node for (NodeList::iterator node = nodeList->begin(); node != nodeList->end(); node++) { if (node->getLinkedData() && !socketMatch(nodeAddress, node->getActiveSocket())) { unsigned char* avatarDataEndpoint = addNodeToBroadcastPacket((unsigned char*)&avatarDataBuffer[0], &*node); int avatarDataLength = avatarDataEndpoint - (unsigned char*)&avatarDataBuffer; if (avatarDataLength + packetLength <= MAX_PACKET_SIZE) { memcpy(currentBufferPosition, &avatarDataBuffer[0], avatarDataLength); packetLength += avatarDataLength; currentBufferPosition += avatarDataLength; } else { packetsSent++; //printf("packetsSent=%d packetLength=%d\n", packetsSent, packetLength); nodeList->getNodeSocket()->send(nodeAddress, broadcastPacket, currentBufferPosition - broadcastPacket); // reset the packet currentBufferPosition = broadcastPacket + numHeaderBytes; packetLength = currentBufferPosition - broadcastPacket; // copy the avatar that didn't fit into the next packet memcpy(currentBufferPosition, &avatarDataBuffer[0], avatarDataLength); packetLength += avatarDataLength; currentBufferPosition += avatarDataLength; } } } packetsSent++; //printf("packetsSent=%d packetLength=%d\n", packetsSent, packetLength); nodeList->getNodeSocket()->send(nodeAddress, broadcastPacket, currentBufferPosition - broadcastPacket); }
int DomainServer::run() { NodeList* nodeList = NodeList::getInstance(); nodeList->addHook(this); ssize_t receivedBytes = 0; char nodeType = '\0'; unsigned char broadcastPacket[MAX_PACKET_SIZE]; unsigned char packetData[MAX_PACKET_SIZE]; unsigned char* currentBufferPos; unsigned char* startPointer; sockaddr_in senderAddress, nodePublicAddress, nodeLocalAddress; nodePublicAddress.sin_family = AF_INET; nodeLocalAddress.sin_family = AF_INET; nodeList->startSilentNodeRemovalThread(); if (!_staticAssignmentFile.exists() || _voxelServerConfig) { if (_voxelServerConfig) { // we have a new VS config, clear the existing file to start fresh _staticAssignmentFile.remove(); } prepopulateStaticAssignmentFile(); } _staticAssignmentFile.open(QIODevice::ReadWrite); _staticAssignmentFileData = _staticAssignmentFile.map(0, _staticAssignmentFile.size()); _staticAssignments = (Assignment*) _staticAssignmentFileData; timeval startTime; gettimeofday(&startTime, NULL); while (true) { while (nodeList->getNodeSocket()->receive((sockaddr *)&senderAddress, packetData, &receivedBytes) && packetVersionMatch(packetData)) { if (packetData[0] == PACKET_TYPE_DOMAIN_REPORT_FOR_DUTY || packetData[0] == PACKET_TYPE_DOMAIN_LIST_REQUEST) { // this is an RFD or domain list request packet, and there is a version match int numBytesSenderHeader = numBytesForPacketHeader(packetData); nodeType = *(packetData + numBytesSenderHeader); int packetIndex = numBytesSenderHeader + sizeof(NODE_TYPE); QUuid nodeUUID = QUuid::fromRfc4122(QByteArray(((char*) packetData + packetIndex), NUM_BYTES_RFC4122_UUID)); packetIndex += NUM_BYTES_RFC4122_UUID; int numBytesPrivateSocket = unpackSocket(packetData + packetIndex, (sockaddr*) &nodePublicAddress); packetIndex += numBytesPrivateSocket; if (nodePublicAddress.sin_addr.s_addr == 0) { // this node wants to use us its STUN server // so set the node public address to whatever we perceive the public address to be nodePublicAddress = senderAddress; // if the sender is on our box then leave its public address to 0 so that // other users attempt to reach it on the same address they have for the domain-server if (senderAddress.sin_addr.s_addr == htonl(INADDR_LOOPBACK)) { nodePublicAddress.sin_addr.s_addr = 0; } } int numBytesPublicSocket = unpackSocket(packetData + packetIndex, (sockaddr*) &nodeLocalAddress); packetIndex += numBytesPublicSocket; const char STATICALLY_ASSIGNED_NODES[3] = { NODE_TYPE_AUDIO_MIXER, NODE_TYPE_AVATAR_MIXER, NODE_TYPE_VOXEL_SERVER }; Assignment* matchingStaticAssignment = NULL; if (memchr(STATICALLY_ASSIGNED_NODES, nodeType, sizeof(STATICALLY_ASSIGNED_NODES)) == NULL || ((matchingStaticAssignment = matchingStaticAssignmentForCheckIn(nodeUUID, nodeType)) || checkInWithUUIDMatchesExistingNode((sockaddr*) &nodePublicAddress, (sockaddr*) &nodeLocalAddress, nodeUUID))) { Node* checkInNode = nodeList->addOrUpdateNode(nodeUUID, nodeType, (sockaddr*) &nodePublicAddress, (sockaddr*) &nodeLocalAddress); if (matchingStaticAssignment) { // this was a newly added node with a matching static assignment if (_hasCompletedRestartHold) { // remove the matching assignment from the assignment queue so we don't take the next check in removeAssignmentFromQueue(matchingStaticAssignment); } // set the linked data for this node to a copy of the matching assignment // so we can re-queue it should the node die Assignment* nodeCopyOfMatchingAssignment = new Assignment(*matchingStaticAssignment); checkInNode->setLinkedData(nodeCopyOfMatchingAssignment); } int numHeaderBytes = populateTypeAndVersion(broadcastPacket, PACKET_TYPE_DOMAIN); currentBufferPos = broadcastPacket + numHeaderBytes; startPointer = currentBufferPos; unsigned char* nodeTypesOfInterest = packetData + packetIndex + sizeof(unsigned char); int numInterestTypes = *(nodeTypesOfInterest - 1); if (numInterestTypes > 0) { // if the node has sent no types of interest, assume they want nothing but their own ID back for (NodeList::iterator node = nodeList->begin(); node != nodeList->end(); node++) { if (!node->matches((sockaddr*) &nodePublicAddress, (sockaddr*) &nodeLocalAddress, nodeType) && memchr(nodeTypesOfInterest, node->getType(), numInterestTypes)) { // don't send avatar nodes to other avatars, that will come from avatar mixer if (nodeType != NODE_TYPE_AGENT || node->getType() != NODE_TYPE_AGENT) { currentBufferPos = addNodeToBroadcastPacket(currentBufferPos, &(*node)); } } } } // update last receive to now uint64_t timeNow = usecTimestampNow(); checkInNode->setLastHeardMicrostamp(timeNow); // send the constructed list back to this node nodeList->getNodeSocket()->send((sockaddr*)&senderAddress, broadcastPacket, (currentBufferPos - startPointer) + numHeaderBytes); } } else if (packetData[0] == PACKET_TYPE_REQUEST_ASSIGNMENT) { qDebug("Received a request for assignment.\n"); if (!_hasCompletedRestartHold) { possiblyAddStaticAssignmentsBackToQueueAfterRestart(&startTime); } if (_assignmentQueue.size() > 0) { // construct the requested assignment from the packet data Assignment requestAssignment(packetData, receivedBytes); Assignment* assignmentToDeploy = deployableAssignmentForRequest(requestAssignment); if (assignmentToDeploy) { // give this assignment out, either the type matches or the requestor said they will take any int numHeaderBytes = populateTypeAndVersion(broadcastPacket, PACKET_TYPE_CREATE_ASSIGNMENT); int numAssignmentBytes = assignmentToDeploy->packToBuffer(broadcastPacket + numHeaderBytes); nodeList->getNodeSocket()->send((sockaddr*) &senderAddress, broadcastPacket, numHeaderBytes + numAssignmentBytes); if (assignmentToDeploy->getNumberOfInstances() == 0) { // there are no more instances of this script to send out, delete it delete assignmentToDeploy; } } } } else if (packetData[0] == PACKET_TYPE_CREATE_ASSIGNMENT) { // this is a create assignment likely recieved from a server needed more clients to help with load // unpack it Assignment* createAssignment = new Assignment(packetData, receivedBytes); qDebug() << "Received a create assignment -" << *createAssignment << "\n"; // make sure we have a matching node with the UUID packed with the assignment // if the node has sent no types of interest, assume they want nothing but their own ID back for (NodeList::iterator node = nodeList->begin(); node != nodeList->end(); node++) { if (node->getLinkedData() && socketMatch((sockaddr*) &senderAddress, node->getPublicSocket()) && ((Assignment*) node->getLinkedData())->getUUID() == createAssignment->getUUID()) { // give the create assignment a new UUID createAssignment->resetUUID(); // add the assignment at the back of the queue _assignmentQueueMutex.lock(); _assignmentQueue.push_back(createAssignment); _assignmentQueueMutex.unlock(); // find the first available spot in the static assignments and put this assignment there for (int i = 0; i < MAX_STATIC_ASSIGNMENT_FILE_ASSIGNMENTS; i++) { if (_staticAssignments[i].getUUID().isNull()) { _staticAssignments[i] = *createAssignment; // we've stuck the assignment in, break out break; } } // we found the matching node that asked for create assignment, break out break; } } } } if (!_hasCompletedRestartHold) { possiblyAddStaticAssignmentsBackToQueueAfterRestart(&startTime); } } this->cleanup(); return 0; }
bool Node::matches(sockaddr* otherPublicSocket, sockaddr* otherLocalSocket, char otherNodeType) { // checks if two node objects are the same node (same type + local + public address) return _type == otherNodeType && socketMatch(_publicSocket, otherPublicSocket) && socketMatch(_localSocket, otherLocalSocket); }