int main(int argc, const char* argv[]) { qInstallMessageHandler(Logging::verboseMessageHandler); NodeList* nodeList = NodeList::createInstance(NODE_TYPE_DOMAIN, DOMAIN_LISTEN_PORT); setvbuf(stdout, NULL, _IOLBF, 0); ssize_t receivedBytes = 0; char nodeType = '\0'; unsigned char broadcastPacket[MAX_PACKET_SIZE]; unsigned char* currentBufferPos; unsigned char* startPointer; sockaddr_in nodePublicAddress, nodeLocalAddress, replyDestinationSocket; nodeLocalAddress.sin_family = AF_INET; in_addr_t serverLocalAddress = getLocalAddress(); nodeList->startSilentNodeRemovalThread(); timeval lastStatSendTime = {}; const char ASSIGNMENT_SERVER_OPTION[] = "-a"; // grab the overriden assignment-server hostname from argv, if it exists const char* customAssignmentServer = getCmdOption(argc, argv, ASSIGNMENT_SERVER_OPTION); if (customAssignmentServer) { sockaddr_in customAssignmentSocket = socketForHostnameAndHostOrderPort(customAssignmentServer, ASSIGNMENT_SERVER_PORT); nodeList->setAssignmentServerSocket((sockaddr*) &customAssignmentSocket); } // use a map to keep track of iterations of silence for assignment creation requests const long long GLOBAL_ASSIGNMENT_REQUEST_INTERVAL_USECS = 1 * 1000 * 1000; timeval lastGlobalAssignmentRequest = {}; // as a domain-server we will always want an audio mixer and avatar mixer // setup the create assignments for those Assignment audioMixerAssignment(Assignment::CreateCommand, Assignment::AudioMixerType, Assignment::LocalLocation); Assignment avatarMixerAssignment(Assignment::CreateCommand, Assignment::AvatarMixerType, Assignment::LocalLocation); // construct a local socket to send with our created assignments to the global AS sockaddr_in localSocket = {}; localSocket.sin_family = AF_INET; localSocket.sin_port = htons(nodeList->getInstance()->getNodeSocket()->getListeningPort()); localSocket.sin_addr.s_addr = serverLocalAddress; // setup the mongoose web server struct mg_context *ctx; struct mg_callbacks callbacks = {}; // list of options. Last element must be NULL. const char *options[] = {"listening_ports", "8080", "document_root", "./resources/web", NULL}; callbacks.begin_request = mongooseRequestHandler; callbacks.upload = mongooseUploadHandler; // Start the web server. ctx = mg_start(&callbacks, NULL, options); while (true) { ::assignmentQueueMutex.lock(); // check if our audio-mixer or avatar-mixer are dead and we don't have existing assignments in the queue // so we can add those assignments back to the front of the queue since they are high-priority if (!nodeList->soloNodeOfType(NODE_TYPE_AVATAR_MIXER) && std::find(::assignmentQueue.begin(), assignmentQueue.end(), &avatarMixerAssignment) == ::assignmentQueue.end()) { qDebug("Missing an avatar mixer and assignment not in queue. Adding.\n"); ::assignmentQueue.push_front(&avatarMixerAssignment); } if (!nodeList->soloNodeOfType(NODE_TYPE_AUDIO_MIXER) && std::find(::assignmentQueue.begin(), ::assignmentQueue.end(), &audioMixerAssignment) == ::assignmentQueue.end()) { qDebug("Missing an audio mixer and assignment not in queue. Adding.\n"); ::assignmentQueue.push_front(&audioMixerAssignment); } ::assignmentQueueMutex.unlock(); while (nodeList->getNodeSocket()->receive((sockaddr *)&nodePublicAddress, packetData, &receivedBytes) && packetVersionMatch(packetData)) { if (packetData[0] == PACKET_TYPE_DOMAIN_REPORT_FOR_DUTY || packetData[0] == PACKET_TYPE_DOMAIN_LIST_REQUEST) { // this is an RFD or domain list request packet, and there is a version match std::map<char, Node *> newestSoloNodes; int numBytesSenderHeader = numBytesForPacketHeader(packetData); nodeType = *(packetData + numBytesSenderHeader); int numBytesSocket = unpackSocket(packetData + numBytesSenderHeader + sizeof(NODE_TYPE), (sockaddr*) &nodeLocalAddress); replyDestinationSocket = nodePublicAddress; // check the node public address // if it matches our local address // or if it's the loopback address we're on the same box if (nodePublicAddress.sin_addr.s_addr == serverLocalAddress || nodePublicAddress.sin_addr.s_addr == htonl(INADDR_LOOPBACK)) { nodePublicAddress.sin_addr.s_addr = 0; } Node* newNode = nodeList->addOrUpdateNode((sockaddr*) &nodePublicAddress, (sockaddr*) &nodeLocalAddress, nodeType, nodeList->getLastNodeID()); // if addOrUpdateNode returns NULL this was a solo node we already have, don't talk back to it if (newNode) { if (newNode->getNodeID() == nodeList->getLastNodeID()) { nodeList->increaseNodeID(); } int numHeaderBytes = populateTypeAndVersion(broadcastPacket, PACKET_TYPE_DOMAIN); currentBufferPos = broadcastPacket + numHeaderBytes; startPointer = currentBufferPos; unsigned char* nodeTypesOfInterest = packetData + numBytesSenderHeader + sizeof(NODE_TYPE) + numBytesSocket + sizeof(unsigned char); int numInterestTypes = *(nodeTypesOfInterest - 1); if (numInterestTypes > 0) { // if the node has sent no types of interest, assume they want nothing but their own ID back for (NodeList::iterator node = nodeList->begin(); node != nodeList->end(); node++) { if (!node->matches((sockaddr*) &nodePublicAddress, (sockaddr*) &nodeLocalAddress, nodeType) && memchr(nodeTypesOfInterest, node->getType(), numInterestTypes)) { // this is not the node themselves // and this is an node of a type in the passed node types of interest // or the node did not pass us any specific types they are interested in if (memchr(SOLO_NODE_TYPES, node->getType(), sizeof(SOLO_NODE_TYPES)) == NULL) { // this is an node of which there can be multiple, just add them to the packet // don't send avatar nodes to other avatars, that will come from avatar mixer if (nodeType != NODE_TYPE_AGENT || node->getType() != NODE_TYPE_AGENT) { currentBufferPos = addNodeToBroadcastPacket(currentBufferPos, &(*node)); } } else { // solo node, we need to only send newest if (newestSoloNodes[node->getType()] == NULL || newestSoloNodes[node->getType()]->getWakeMicrostamp() < node->getWakeMicrostamp()) { // we have to set the newer solo node to add it to the broadcast later newestSoloNodes[node->getType()] = &(*node); } } } } for (std::map<char, Node *>::iterator soloNode = newestSoloNodes.begin(); soloNode != newestSoloNodes.end(); soloNode++) { // this is the newest alive solo node, add them to the packet currentBufferPos = addNodeToBroadcastPacket(currentBufferPos, soloNode->second); } } // update last receive to now uint64_t timeNow = usecTimestampNow(); newNode->setLastHeardMicrostamp(timeNow); if (packetData[0] == PACKET_TYPE_DOMAIN_REPORT_FOR_DUTY && memchr(SOLO_NODE_TYPES, nodeType, sizeof(SOLO_NODE_TYPES))) { newNode->setWakeMicrostamp(timeNow); } // add the node ID to the end of the pointer currentBufferPos += packNodeId(currentBufferPos, newNode->getNodeID()); // send the constructed list back to this node nodeList->getNodeSocket()->send((sockaddr*)&replyDestinationSocket, broadcastPacket, (currentBufferPos - startPointer) + numHeaderBytes); } } else if (packetData[0] == PACKET_TYPE_REQUEST_ASSIGNMENT) { qDebug("Received a request for assignment.\n"); ::assignmentQueueMutex.lock(); // this is an unassigned client talking to us directly for an assignment // go through our queue and see if there are any assignments to give out std::deque<Assignment*>::iterator assignment = ::assignmentQueue.begin(); while (assignment != ::assignmentQueue.end()) { // give this assignment out, no conditions stop us from giving it to the local assignment client int numHeaderBytes = populateTypeAndVersion(broadcastPacket, PACKET_TYPE_CREATE_ASSIGNMENT); int numAssignmentBytes = (*assignment)->packToBuffer(broadcastPacket + numHeaderBytes); nodeList->getNodeSocket()->send((sockaddr*) &nodePublicAddress, broadcastPacket, numHeaderBytes + numAssignmentBytes); // remove the assignment from the queue ::assignmentQueue.erase(assignment); if ((*assignment)->getType() == Assignment::AgentType) { // if this is a script assignment we need to delete it to avoid a memory leak delete *assignment; } // stop looping, we've handed out an assignment break; } ::assignmentQueueMutex.unlock(); } } // if ASSIGNMENT_REQUEST_INTERVAL_USECS have passed since last global assignment request then fire off another if (usecTimestampNow() - usecTimestamp(&lastGlobalAssignmentRequest) >= GLOBAL_ASSIGNMENT_REQUEST_INTERVAL_USECS) { gettimeofday(&lastGlobalAssignmentRequest, NULL); ::assignmentQueueMutex.lock(); // go through our queue and see if there are any assignments to send to the global assignment server std::deque<Assignment*>::iterator assignment = ::assignmentQueue.begin(); while (assignment != assignmentQueue.end()) { if ((*assignment)->getLocation() != Assignment::LocalLocation) { // attach our local socket to the assignment so the assignment-server can optionally hand it out (*assignment)->setAttachedLocalSocket((sockaddr*) &localSocket); nodeList->sendAssignment(*(*assignment)); // remove the assignment from the queue ::assignmentQueue.erase(assignment); if ((*assignment)->getType() == Assignment::AgentType) { // if this is a script assignment we need to delete it to avoid a memory leak delete *assignment; } // stop looping, we've handed out an assignment break; } else { // push forward the iterator to check the next assignment assignment++; } } ::assignmentQueueMutex.unlock(); } if (Logging::shouldSendStats()) { if (usecTimestampNow() - usecTimestamp(&lastStatSendTime) >= (NODE_COUNT_STAT_INTERVAL_MSECS * 1000)) { // time to send our count of nodes and servers to logstash const char NODE_COUNT_LOGSTASH_KEY[] = "ds-node-count"; Logging::stashValue(STAT_TYPE_TIMER, NODE_COUNT_LOGSTASH_KEY, nodeList->getNumAliveNodes()); gettimeofday(&lastStatSendTime, NULL); } } } return 0; }
int DomainServer::run() { NodeList* nodeList = NodeList::getInstance(); nodeList->addHook(this); ssize_t receivedBytes = 0; char nodeType = '\0'; unsigned char broadcastPacket[MAX_PACKET_SIZE]; unsigned char packetData[MAX_PACKET_SIZE]; unsigned char* currentBufferPos; unsigned char* startPointer; sockaddr_in nodePublicAddress, nodeLocalAddress, replyDestinationSocket; nodeLocalAddress.sin_family = AF_INET; in_addr_t serverLocalAddress = getLocalAddress(); nodeList->startSilentNodeRemovalThread(); if (!_staticAssignmentFile.exists() || _voxelServerConfig) { if (_voxelServerConfig) { // we have a new VS config, clear the existing file to start fresh _staticAssignmentFile.remove(); } prepopulateStaticAssignmentFile(); } _staticAssignmentFile.open(QIODevice::ReadWrite); _staticAssignmentFileData = _staticAssignmentFile.map(0, _staticAssignmentFile.size()); _staticAssignments = (Assignment*) _staticAssignmentFileData; timeval startTime; gettimeofday(&startTime, NULL); while (true) { while (nodeList->getNodeSocket()->receive((sockaddr *)&nodePublicAddress, packetData, &receivedBytes) && packetVersionMatch(packetData)) { if (packetData[0] == PACKET_TYPE_DOMAIN_REPORT_FOR_DUTY || packetData[0] == PACKET_TYPE_DOMAIN_LIST_REQUEST) { // this is an RFD or domain list request packet, and there is a version match int numBytesSenderHeader = numBytesForPacketHeader(packetData); nodeType = *(packetData + numBytesSenderHeader); int numBytesSocket = unpackSocket(packetData + numBytesSenderHeader + sizeof(NODE_TYPE), (sockaddr*) &nodeLocalAddress); replyDestinationSocket = nodePublicAddress; // check the node public address // if it matches our local address // or if it's the loopback address we're on the same box if (nodePublicAddress.sin_addr.s_addr == serverLocalAddress || nodePublicAddress.sin_addr.s_addr == htonl(INADDR_LOOPBACK)) { nodePublicAddress.sin_addr.s_addr = 0; } const char STATICALLY_ASSIGNED_NODES[3] = { NODE_TYPE_AUDIO_MIXER, NODE_TYPE_AVATAR_MIXER, NODE_TYPE_VOXEL_SERVER }; Assignment* matchingStaticAssignment = NULL; if (memchr(STATICALLY_ASSIGNED_NODES, nodeType, sizeof(STATICALLY_ASSIGNED_NODES)) == NULL || ((matchingStaticAssignment = matchingStaticAssignmentForCheckIn(nodeType, packetData)) || checkInWithUUIDMatchesExistingNode((sockaddr*) &nodePublicAddress, (sockaddr*) &nodeLocalAddress, packetData))) { Node* checkInNode = nodeList->addOrUpdateNode((sockaddr*) &nodePublicAddress, (sockaddr*) &nodeLocalAddress, nodeType, nodeList->getLastNodeID()); if (matchingStaticAssignment) { // this was a newly added node with a matching static assignment if (_hasCompletedRestartHold) { // remove the matching assignment from the assignment queue so we don't take the next check in removeAssignmentFromQueue(matchingStaticAssignment); } // set the linked data for this node to a copy of the matching assignment // so we can re-queue it should the node die Assignment* nodeCopyOfMatchingAssignment = new Assignment(*matchingStaticAssignment); checkInNode->setLinkedData(nodeCopyOfMatchingAssignment); } int numHeaderBytes = populateTypeAndVersion(broadcastPacket, PACKET_TYPE_DOMAIN); currentBufferPos = broadcastPacket + numHeaderBytes; startPointer = currentBufferPos; int numBytesUUID = (nodeType == NODE_TYPE_AUDIO_MIXER || nodeType == NODE_TYPE_AVATAR_MIXER) ? NUM_BYTES_RFC4122_UUID : 0; unsigned char* nodeTypesOfInterest = packetData + numBytesSenderHeader + numBytesUUID + sizeof(NODE_TYPE) + numBytesSocket + sizeof(unsigned char); int numInterestTypes = *(nodeTypesOfInterest - 1); if (numInterestTypes > 0) { // if the node has sent no types of interest, assume they want nothing but their own ID back for (NodeList::iterator node = nodeList->begin(); node != nodeList->end(); node++) { if (!node->matches((sockaddr*) &nodePublicAddress, (sockaddr*) &nodeLocalAddress, nodeType) && memchr(nodeTypesOfInterest, node->getType(), numInterestTypes)) { // don't send avatar nodes to other avatars, that will come from avatar mixer if (nodeType != NODE_TYPE_AGENT || node->getType() != NODE_TYPE_AGENT) { currentBufferPos = addNodeToBroadcastPacket(currentBufferPos, &(*node)); } } } } // update last receive to now uint64_t timeNow = usecTimestampNow(); checkInNode->setLastHeardMicrostamp(timeNow); // add the node ID to the end of the pointer currentBufferPos += packNodeId(currentBufferPos, checkInNode->getNodeID()); // send the constructed list back to this node nodeList->getNodeSocket()->send((sockaddr*)&replyDestinationSocket, broadcastPacket, (currentBufferPos - startPointer) + numHeaderBytes); } } else if (packetData[0] == PACKET_TYPE_REQUEST_ASSIGNMENT) { qDebug("Received a request for assignment.\n"); if (!_hasCompletedRestartHold) { possiblyAddStaticAssignmentsBackToQueueAfterRestart(&startTime); } if (_assignmentQueue.size() > 0) { // construct the requested assignment from the packet data Assignment requestAssignment(packetData, receivedBytes); Assignment* assignmentToDeploy = deployableAssignmentForRequest(requestAssignment); if (assignmentToDeploy) { // give this assignment out, either the type matches or the requestor said they will take any int numHeaderBytes = populateTypeAndVersion(broadcastPacket, PACKET_TYPE_CREATE_ASSIGNMENT); int numAssignmentBytes = assignmentToDeploy->packToBuffer(broadcastPacket + numHeaderBytes); nodeList->getNodeSocket()->send((sockaddr*) &nodePublicAddress, broadcastPacket, numHeaderBytes + numAssignmentBytes); } } } else if (packetData[0] == PACKET_TYPE_CREATE_ASSIGNMENT) { // this is a create assignment likely recieved from a server needed more clients to help with load // unpack it Assignment* createAssignment = new Assignment(packetData, receivedBytes); qDebug() << "Received a create assignment -" << *createAssignment << "\n"; // check the node public address // if it matches our local address // or if it's the loopback address we're on the same box if (nodePublicAddress.sin_addr.s_addr == serverLocalAddress || nodePublicAddress.sin_addr.s_addr == htonl(INADDR_LOOPBACK)) { nodePublicAddress.sin_addr.s_addr = 0; } // make sure we have a matching node with the UUID packed with the assignment // if the node has sent no types of interest, assume they want nothing but their own ID back for (NodeList::iterator node = nodeList->begin(); node != nodeList->end(); node++) { if (node->getLinkedData() && socketMatch((sockaddr*) &nodePublicAddress, node->getPublicSocket()) && ((Assignment*) node->getLinkedData())->getUUID() == createAssignment->getUUID()) { // give the create assignment a new UUID createAssignment->resetUUID(); // add the assignment at the back of the queue _assignmentQueueMutex.lock(); _assignmentQueue.push_back(createAssignment); _assignmentQueueMutex.unlock(); // find the first available spot in the static assignments and put this assignment there for (int i = 0; i < MAX_STATIC_ASSIGNMENT_FILE_ASSIGNMENTS; i++) { if (_staticAssignments[i].getUUID().isNull()) { _staticAssignments[i] = *createAssignment; // we've stuck the assignment in, break out break; } } // we found the matching node that asked for create assignment, break out break; } } } } if (!_hasCompletedRestartHold) { possiblyAddStaticAssignmentsBackToQueueAfterRestart(&startTime); } } this->cleanup(); return 0; }
int main(int argc, const char * argv[]) { NodeList* nodeList = NodeList::createInstance(NODE_TYPE_DOMAIN, DOMAIN_LISTEN_PORT); // If user asks to run in "local" mode then we do NOT replace the IP // with the EC2 IP. Otherwise, we will replace the IP like we used to // this allows developers to run a local domain without recompiling the // domain server bool isLocalMode = cmdOptionExists(argc, argv, "--local"); if (isLocalMode) { printf("NOTE: Running in Local Mode!\n"); } else { printf("--------------------------------------------------\n"); printf("NOTE: Running in EC2 Mode. \n"); printf("If you're a developer testing a local system, you\n"); printf("probably want to include --local on command line.\n"); printf("--------------------------------------------------\n"); } setvbuf(stdout, NULL, _IOLBF, 0); ssize_t receivedBytes = 0; char nodeType = '\0'; unsigned char broadcastPacket[MAX_PACKET_SIZE]; int numHeaderBytes = populateTypeAndVersion(broadcastPacket, PACKET_TYPE_DOMAIN); unsigned char* currentBufferPos; unsigned char* startPointer; sockaddr_in nodePublicAddress, nodeLocalAddress; nodeLocalAddress.sin_family = AF_INET; in_addr_t serverLocalAddress = getLocalAddress(); nodeList->startSilentNodeRemovalThread(); timeval lastStatSendTime = {}; while (true) { if (nodeList->getNodeSocket()->receive((sockaddr *)&nodePublicAddress, packetData, &receivedBytes) && (packetData[0] == PACKET_TYPE_DOMAIN_REPORT_FOR_DUTY || packetData[0] == PACKET_TYPE_DOMAIN_LIST_REQUEST) && packetVersionMatch(packetData)) { // this is an RFD or domain list request packet, and there is a version match std::map<char, Node *> newestSoloNodes; int numBytesSenderHeader = numBytesForPacketHeader(packetData); nodeType = *(packetData + numBytesSenderHeader); int numBytesSocket = unpackSocket(packetData + numBytesSenderHeader + sizeof(NODE_TYPE), (sockaddr*) &nodeLocalAddress); sockaddr* destinationSocket = (sockaddr*) &nodePublicAddress; // check the node public address // if it matches our local address we're on the same box // so hardcode the EC2 public address for now if (nodePublicAddress.sin_addr.s_addr == serverLocalAddress) { // If we're not running "local" then we do replace the IP // with the EC2 IP. Otherwise, we use our normal public IP if (!isLocalMode) { nodePublicAddress.sin_addr.s_addr = 895283510; // local IP in this format... destinationSocket = (sockaddr*) &nodeLocalAddress; } } Node* newNode = nodeList->addOrUpdateNode((sockaddr*) &nodePublicAddress, (sockaddr*) &nodeLocalAddress, nodeType, nodeList->getLastNodeID()); if (newNode->getNodeID() == nodeList->getLastNodeID()) { nodeList->increaseNodeID(); } currentBufferPos = broadcastPacket + numHeaderBytes; startPointer = currentBufferPos; unsigned char* nodeTypesOfInterest = packetData + numBytesSenderHeader + sizeof(NODE_TYPE) + numBytesSocket + sizeof(unsigned char); int numInterestTypes = *(nodeTypesOfInterest - 1); if (numInterestTypes > 0) { // if the node has sent no types of interest, assume they want nothing but their own ID back for (NodeList::iterator node = nodeList->begin(); node != nodeList->end(); node++) { if (!node->matches((sockaddr*) &nodePublicAddress, (sockaddr*) &nodeLocalAddress, nodeType) && memchr(nodeTypesOfInterest, node->getType(), numInterestTypes)) { // this is not the node themselves // and this is an node of a type in the passed node types of interest // or the node did not pass us any specific types they are interested in if (memchr(SOLO_NODE_TYPES, node->getType(), sizeof(SOLO_NODE_TYPES)) == NULL) { // this is an node of which there can be multiple, just add them to the packet // don't send avatar nodes to other avatars, that will come from avatar mixer if (nodeType != NODE_TYPE_AGENT || node->getType() != NODE_TYPE_AGENT) { currentBufferPos = addNodeToBroadcastPacket(currentBufferPos, &(*node)); } } else { // solo node, we need to only send newest if (newestSoloNodes[node->getType()] == NULL || newestSoloNodes[node->getType()]->getWakeMicrostamp() < node->getWakeMicrostamp()) { // we have to set the newer solo node to add it to the broadcast later newestSoloNodes[node->getType()] = &(*node); } } } } for (std::map<char, Node *>::iterator soloNode = newestSoloNodes.begin(); soloNode != newestSoloNodes.end(); soloNode++) { // this is the newest alive solo node, add them to the packet currentBufferPos = addNodeToBroadcastPacket(currentBufferPos, soloNode->second); } } // update last receive to now uint64_t timeNow = usecTimestampNow(); newNode->setLastHeardMicrostamp(timeNow); if (packetData[0] == PACKET_TYPE_DOMAIN_REPORT_FOR_DUTY && memchr(SOLO_NODE_TYPES, nodeType, sizeof(SOLO_NODE_TYPES))) { newNode->setWakeMicrostamp(timeNow); } // add the node ID to the end of the pointer currentBufferPos += packNodeId(currentBufferPos, newNode->getNodeID()); // send the constructed list back to this node nodeList->getNodeSocket()->send(destinationSocket, broadcastPacket, (currentBufferPos - startPointer) + numHeaderBytes); } if (Logstash::shouldSendStats()) { if (usecTimestampNow() - usecTimestamp(&lastStatSendTime) >= (NODE_COUNT_STAT_INTERVAL_MSECS * 1000)) { // time to send our count of nodes and servers to logstash const char NODE_COUNT_LOGSTASH_KEY[] = "ds-node-count"; Logstash::stashValue(STAT_TYPE_TIMER, NODE_COUNT_LOGSTASH_KEY, nodeList->getNumAliveNodes()); gettimeofday(&lastStatSendTime, NULL); } } } return 0; }