Пример #1
0
Node* NodeList::addOrUpdateNode(sockaddr* publicSocket, sockaddr* localSocket, char nodeType, uint16_t nodeId) {
    NodeList::iterator node = end();
    
    if (publicSocket) {
        for (node = begin(); node != end(); node++) {
            if (node->matches(publicSocket, localSocket, nodeType)) {
                // we already have this node, stop checking
                break;
            }
        }
    }
    
    if (node == end()) {
        // if we already had this node AND it's a solo type then bust out of here
        if (soloNodeOfType(nodeType)) {
            return NULL;
        }
        
        // we didn't have this node, so add them
        Node* newNode = new Node(publicSocket, localSocket, nodeType, nodeId);
        
        if (socketMatch(publicSocket, localSocket)) {
            // likely debugging scenario with two nodes on local network
            // set the node active right away
            newNode->activatePublicSocket();
        }
   
        if (newNode->getType() == NODE_TYPE_VOXEL_SERVER ||
            newNode->getType() == NODE_TYPE_AVATAR_MIXER ||
            newNode->getType() == NODE_TYPE_AUDIO_MIXER) {
            // this is currently the cheat we use to talk directly to our test servers on EC2
            // to be removed when we have a proper identification strategy
            newNode->activatePublicSocket();
        }
        
        addNodeToList(newNode);
        
        return newNode;
    } else {
        
        if (node->getType() == NODE_TYPE_AUDIO_MIXER ||
            node->getType() == NODE_TYPE_VOXEL_SERVER) {
            // until the Audio class also uses our nodeList, we need to update
            // the lastRecvTimeUsecs for the audio mixer so it doesn't get killed and re-added continously
            node->setLastHeardMicrostamp(usecTimestampNow());
        }
        
        // we had this node already, do nothing for now
        return &*node;
    }    
}
Пример #2
0
int DomainServer::run() {
    NodeList* nodeList = NodeList::getInstance();
    
    nodeList->addHook(this);
    
    ssize_t receivedBytes = 0;
    char nodeType = '\0';
    
    unsigned char broadcastPacket[MAX_PACKET_SIZE];
    unsigned char packetData[MAX_PACKET_SIZE];
    
    unsigned char* currentBufferPos;
    unsigned char* startPointer;
    
    sockaddr_in senderAddress, nodePublicAddress, nodeLocalAddress;
    nodePublicAddress.sin_family = AF_INET;
    nodeLocalAddress.sin_family = AF_INET;
    
    nodeList->startSilentNodeRemovalThread();
    
    if (!_staticAssignmentFile.exists() || _voxelServerConfig) {
        
        if (_voxelServerConfig) {
            // we have a new VS config, clear the existing file to start fresh
            _staticAssignmentFile.remove();
        }
        
        prepopulateStaticAssignmentFile();
    }
    
    _staticAssignmentFile.open(QIODevice::ReadWrite);
    
    _staticAssignmentFileData = _staticAssignmentFile.map(0, _staticAssignmentFile.size());
    
    _staticAssignments = (Assignment*) _staticAssignmentFileData;
    
    timeval startTime;
    gettimeofday(&startTime, NULL);
    
    while (true) {
        while (nodeList->getNodeSocket()->receive((sockaddr *)&senderAddress, packetData, &receivedBytes) &&
               packetVersionMatch(packetData)) {
            if (packetData[0] == PACKET_TYPE_DOMAIN_REPORT_FOR_DUTY || packetData[0] == PACKET_TYPE_DOMAIN_LIST_REQUEST) {
                // this is an RFD or domain list request packet, and there is a version match
                
                int numBytesSenderHeader = numBytesForPacketHeader(packetData);
                
                nodeType = *(packetData + numBytesSenderHeader);
                
                int packetIndex = numBytesSenderHeader + sizeof(NODE_TYPE);
                QUuid nodeUUID = QUuid::fromRfc4122(QByteArray(((char*) packetData + packetIndex), NUM_BYTES_RFC4122_UUID));
                packetIndex += NUM_BYTES_RFC4122_UUID;
                
                int numBytesPrivateSocket = unpackSocket(packetData + packetIndex, (sockaddr*) &nodePublicAddress);
                packetIndex += numBytesPrivateSocket;
                
                if (nodePublicAddress.sin_addr.s_addr == 0) {
                    // this node wants to use us its STUN server
                    // so set the node public address to whatever we perceive the public address to be
                    
                    nodePublicAddress = senderAddress;
                    
                    // if the sender is on our box then leave its public address to 0 so that
                    // other users attempt to reach it on the same address they have for the domain-server
                    if (senderAddress.sin_addr.s_addr == htonl(INADDR_LOOPBACK)) {
                        nodePublicAddress.sin_addr.s_addr = 0;
                    }
                }
                
                int numBytesPublicSocket = unpackSocket(packetData + packetIndex, (sockaddr*) &nodeLocalAddress);
                packetIndex += numBytesPublicSocket;
                
                const char STATICALLY_ASSIGNED_NODES[3] = {
                    NODE_TYPE_AUDIO_MIXER,
                    NODE_TYPE_AVATAR_MIXER,
                    NODE_TYPE_VOXEL_SERVER
                };
                
                Assignment* matchingStaticAssignment = NULL;
                
                if (memchr(STATICALLY_ASSIGNED_NODES, nodeType, sizeof(STATICALLY_ASSIGNED_NODES)) == NULL
                    || ((matchingStaticAssignment = matchingStaticAssignmentForCheckIn(nodeUUID, nodeType))
                        || checkInWithUUIDMatchesExistingNode((sockaddr*) &nodePublicAddress,
                                                              (sockaddr*) &nodeLocalAddress,
                                                              nodeUUID)))
                {
                    Node* checkInNode = nodeList->addOrUpdateNode(nodeUUID,
                                                                  nodeType,
                                                                  (sockaddr*) &nodePublicAddress,
                                                                  (sockaddr*) &nodeLocalAddress);
                    
                    if (matchingStaticAssignment) {
                        // this was a newly added node with a matching static assignment
                        
                        if (_hasCompletedRestartHold) {
                            // remove the matching assignment from the assignment queue so we don't take the next check in
                            removeAssignmentFromQueue(matchingStaticAssignment);
                        }
                        
                        // set the linked data for this node to a copy of the matching assignment
                        // so we can re-queue it should the node die
                        Assignment* nodeCopyOfMatchingAssignment = new Assignment(*matchingStaticAssignment);
                
                        checkInNode->setLinkedData(nodeCopyOfMatchingAssignment);
                    }
                    
                    int numHeaderBytes = populateTypeAndVersion(broadcastPacket, PACKET_TYPE_DOMAIN);
                    
                    currentBufferPos = broadcastPacket + numHeaderBytes;
                    startPointer = currentBufferPos;
                    
                    unsigned char* nodeTypesOfInterest = packetData + packetIndex + sizeof(unsigned char);
                    int numInterestTypes = *(nodeTypesOfInterest - 1);
                    
                    if (numInterestTypes > 0) {
                        // if the node has sent no types of interest, assume they want nothing but their own ID back
                        for (NodeList::iterator node = nodeList->begin(); node != nodeList->end(); node++) {
                            if (!node->matches((sockaddr*) &nodePublicAddress, (sockaddr*) &nodeLocalAddress, nodeType) &&
                                memchr(nodeTypesOfInterest, node->getType(), numInterestTypes)) {
                                
                                // don't send avatar nodes to other avatars, that will come from avatar mixer
                                if (nodeType != NODE_TYPE_AGENT || node->getType() != NODE_TYPE_AGENT) {
                                    currentBufferPos = addNodeToBroadcastPacket(currentBufferPos, &(*node));
                                }
                                
                            }
                        }
                    }
                    
                    // update last receive to now
                    uint64_t timeNow = usecTimestampNow();
                    checkInNode->setLastHeardMicrostamp(timeNow);
                    
                    // send the constructed list back to this node
                    nodeList->getNodeSocket()->send((sockaddr*)&senderAddress,
                                                    broadcastPacket,
                                                    (currentBufferPos - startPointer) + numHeaderBytes);
                }
            } else if (packetData[0] == PACKET_TYPE_REQUEST_ASSIGNMENT) {
                
                qDebug("Received a request for assignment.\n");
                
                if (!_hasCompletedRestartHold) {
                    possiblyAddStaticAssignmentsBackToQueueAfterRestart(&startTime);
                }
                
                if (_assignmentQueue.size() > 0) {
                    // construct the requested assignment from the packet data
                    Assignment requestAssignment(packetData, receivedBytes);
                    
                    Assignment* assignmentToDeploy = deployableAssignmentForRequest(requestAssignment);
                    
                    if (assignmentToDeploy) {
                    
                        // give this assignment out, either the type matches or the requestor said they will take any
                        int numHeaderBytes = populateTypeAndVersion(broadcastPacket, PACKET_TYPE_CREATE_ASSIGNMENT);
                        int numAssignmentBytes = assignmentToDeploy->packToBuffer(broadcastPacket + numHeaderBytes);
        
                        nodeList->getNodeSocket()->send((sockaddr*) &senderAddress,
                                                        broadcastPacket,
                                                        numHeaderBytes + numAssignmentBytes);
                        
                        if (assignmentToDeploy->getNumberOfInstances() == 0) {
                            // there are no more instances of this script to send out, delete it
                            delete assignmentToDeploy;
                        }
                    }
                    
                }
            } else if (packetData[0] == PACKET_TYPE_CREATE_ASSIGNMENT) {
                // this is a create assignment likely recieved from a server needed more clients to help with load
                
                // unpack it
                Assignment* createAssignment = new Assignment(packetData, receivedBytes);
                
                qDebug() << "Received a create assignment -" << *createAssignment << "\n";
                
                // make sure we have a matching node with the UUID packed with the assignment
                // if the node has sent no types of interest, assume they want nothing but their own ID back
                for (NodeList::iterator node = nodeList->begin(); node != nodeList->end(); node++) {
                    if (node->getLinkedData()
                        && socketMatch((sockaddr*) &senderAddress, node->getPublicSocket())
                        && ((Assignment*) node->getLinkedData())->getUUID() == createAssignment->getUUID()) {
                        
                        // give the create assignment a new UUID
                        createAssignment->resetUUID();
                        
                        // add the assignment at the back of the queue
                        _assignmentQueueMutex.lock();
                        _assignmentQueue.push_back(createAssignment);
                        _assignmentQueueMutex.unlock();
                        
                        // find the first available spot in the static assignments and put this assignment there
                        for (int i = 0; i < MAX_STATIC_ASSIGNMENT_FILE_ASSIGNMENTS; i++) {
                            if (_staticAssignments[i].getUUID().isNull()) {
                                _staticAssignments[i] = *createAssignment;
                                
                                // we've stuck the assignment in, break out
                                break;
                            }
                        }
                        
                        // we found the matching node that asked for create assignment, break out
                        break;
                    }
                }
            }
        }
        
        if (!_hasCompletedRestartHold) {
            possiblyAddStaticAssignmentsBackToQueueAfterRestart(&startTime);
        }
    }
    
    this->cleanup();
    
    return 0;
}
Пример #3
0
int main(int argc, const char* argv[]) {
    
    qInstallMessageHandler(Logging::verboseMessageHandler);
    
    NodeList* nodeList = NodeList::createInstance(NODE_TYPE_DOMAIN, DOMAIN_LISTEN_PORT);

    setvbuf(stdout, NULL, _IOLBF, 0);
    
    ssize_t receivedBytes = 0;
    char nodeType = '\0';
    
    unsigned char broadcastPacket[MAX_PACKET_SIZE];
    
    unsigned char* currentBufferPos;
    unsigned char* startPointer;
    
    sockaddr_in nodePublicAddress, nodeLocalAddress, replyDestinationSocket;
    nodeLocalAddress.sin_family = AF_INET;
    
    in_addr_t serverLocalAddress = getLocalAddress();
    
    nodeList->startSilentNodeRemovalThread();
    
    timeval lastStatSendTime = {};
    const char ASSIGNMENT_SERVER_OPTION[] = "-a";
    
    // grab the overriden assignment-server hostname from argv, if it exists
    const char* customAssignmentServer = getCmdOption(argc, argv, ASSIGNMENT_SERVER_OPTION);
    if (customAssignmentServer) {
        sockaddr_in customAssignmentSocket = socketForHostnameAndHostOrderPort(customAssignmentServer, ASSIGNMENT_SERVER_PORT);
        nodeList->setAssignmentServerSocket((sockaddr*) &customAssignmentSocket);
    }
    
    // use a map to keep track of iterations of silence for assignment creation requests
    const long long GLOBAL_ASSIGNMENT_REQUEST_INTERVAL_USECS = 1 * 1000 * 1000;
    timeval lastGlobalAssignmentRequest = {};
    
    // as a domain-server we will always want an audio mixer and avatar mixer
    // setup the create assignments for those
    Assignment audioMixerAssignment(Assignment::CreateCommand,
                                    Assignment::AudioMixerType,
                                    Assignment::LocalLocation);
    
    Assignment avatarMixerAssignment(Assignment::CreateCommand,
                                     Assignment::AvatarMixerType,
                                     Assignment::LocalLocation);
    
    // construct a local socket to send with our created assignments to the global AS
    sockaddr_in localSocket = {};
    localSocket.sin_family = AF_INET;
    localSocket.sin_port = htons(nodeList->getInstance()->getNodeSocket()->getListeningPort());
    localSocket.sin_addr.s_addr = serverLocalAddress;
    
    // setup the mongoose web server
    struct mg_context *ctx;
    struct mg_callbacks callbacks = {};
    
    // list of options. Last element must be NULL.
    const char *options[] = {"listening_ports", "8080",
                             "document_root", "./resources/web", NULL};
    
    callbacks.begin_request = mongooseRequestHandler;
    callbacks.upload = mongooseUploadHandler;
    
    // Start the web server.
    ctx = mg_start(&callbacks, NULL, options);
    
    while (true) {
        
        ::assignmentQueueMutex.lock();
        // check if our audio-mixer or avatar-mixer are dead and we don't have existing assignments in the queue
        // so we can add those assignments back to the front of the queue since they are high-priority
        if (!nodeList->soloNodeOfType(NODE_TYPE_AVATAR_MIXER) &&
            std::find(::assignmentQueue.begin(), assignmentQueue.end(), &avatarMixerAssignment) == ::assignmentQueue.end()) {
            qDebug("Missing an avatar mixer and assignment not in queue. Adding.\n");
            ::assignmentQueue.push_front(&avatarMixerAssignment);
        }
        
        if (!nodeList->soloNodeOfType(NODE_TYPE_AUDIO_MIXER) &&
            std::find(::assignmentQueue.begin(), ::assignmentQueue.end(), &audioMixerAssignment) == ::assignmentQueue.end()) {
            qDebug("Missing an audio mixer and assignment not in queue. Adding.\n");
            ::assignmentQueue.push_front(&audioMixerAssignment);
        }
        ::assignmentQueueMutex.unlock();
        
        while (nodeList->getNodeSocket()->receive((sockaddr *)&nodePublicAddress, packetData, &receivedBytes) &&
               packetVersionMatch(packetData)) {
            if (packetData[0] == PACKET_TYPE_DOMAIN_REPORT_FOR_DUTY || packetData[0] == PACKET_TYPE_DOMAIN_LIST_REQUEST) {
                // this is an RFD or domain list request packet, and there is a version match
                std::map<char, Node *> newestSoloNodes;
                
                int numBytesSenderHeader = numBytesForPacketHeader(packetData);
                
                nodeType = *(packetData + numBytesSenderHeader);
                int numBytesSocket = unpackSocket(packetData + numBytesSenderHeader + sizeof(NODE_TYPE),
                                                  (sockaddr*) &nodeLocalAddress);
                
                replyDestinationSocket = nodePublicAddress;
                
                // check the node public address
                // if it matches our local address
                // or if it's the loopback address we're on the same box
                if (nodePublicAddress.sin_addr.s_addr == serverLocalAddress ||
                    nodePublicAddress.sin_addr.s_addr == htonl(INADDR_LOOPBACK)) {
                    
                    nodePublicAddress.sin_addr.s_addr = 0;
                }
                
                Node* newNode = nodeList->addOrUpdateNode((sockaddr*) &nodePublicAddress,
                                                          (sockaddr*) &nodeLocalAddress,
                                                          nodeType,
                                                          nodeList->getLastNodeID());
                
                // if addOrUpdateNode returns NULL this was a solo node we already have, don't talk back to it
                if (newNode) {
                    if (newNode->getNodeID() == nodeList->getLastNodeID()) {
                        nodeList->increaseNodeID();
                    }
                    
                    int numHeaderBytes = populateTypeAndVersion(broadcastPacket, PACKET_TYPE_DOMAIN);
                    
                    currentBufferPos = broadcastPacket + numHeaderBytes;
                    startPointer = currentBufferPos;
                    
                    unsigned char* nodeTypesOfInterest = packetData + numBytesSenderHeader + sizeof(NODE_TYPE)
                    + numBytesSocket + sizeof(unsigned char);
                    int numInterestTypes = *(nodeTypesOfInterest - 1);
                    
                    if (numInterestTypes > 0) {
                        // if the node has sent no types of interest, assume they want nothing but their own ID back
                        for (NodeList::iterator node = nodeList->begin(); node != nodeList->end(); node++) {
                            if (!node->matches((sockaddr*) &nodePublicAddress, (sockaddr*) &nodeLocalAddress, nodeType) &&
                                memchr(nodeTypesOfInterest, node->getType(), numInterestTypes)) {
                                // this is not the node themselves
                                // and this is an node of a type in the passed node types of interest
                                // or the node did not pass us any specific types they are interested in
                                
                                if (memchr(SOLO_NODE_TYPES, node->getType(), sizeof(SOLO_NODE_TYPES)) == NULL) {
                                    // this is an node of which there can be multiple, just add them to the packet
                                    // don't send avatar nodes to other avatars, that will come from avatar mixer
                                    if (nodeType != NODE_TYPE_AGENT || node->getType() != NODE_TYPE_AGENT) {
                                        currentBufferPos = addNodeToBroadcastPacket(currentBufferPos, &(*node));
                                    }
                                    
                                } else {
                                    // solo node, we need to only send newest
                                    if (newestSoloNodes[node->getType()] == NULL ||
                                        newestSoloNodes[node->getType()]->getWakeMicrostamp() < node->getWakeMicrostamp()) {
                                        // we have to set the newer solo node to add it to the broadcast later
                                        newestSoloNodes[node->getType()] = &(*node);
                                    }
                                }
                            }
                        }
                        
                        for (std::map<char, Node *>::iterator soloNode = newestSoloNodes.begin();
                             soloNode != newestSoloNodes.end();
                             soloNode++) {
                            // this is the newest alive solo node, add them to the packet
                            currentBufferPos = addNodeToBroadcastPacket(currentBufferPos, soloNode->second);
                        }
                    }
                    
                    // update last receive to now
                    uint64_t timeNow = usecTimestampNow();
                    newNode->setLastHeardMicrostamp(timeNow);
                    
                    if (packetData[0] == PACKET_TYPE_DOMAIN_REPORT_FOR_DUTY
                        && memchr(SOLO_NODE_TYPES, nodeType, sizeof(SOLO_NODE_TYPES))) {
                        newNode->setWakeMicrostamp(timeNow);
                    }
                    
                    // add the node ID to the end of the pointer
                    currentBufferPos += packNodeId(currentBufferPos, newNode->getNodeID());
                    
                    // send the constructed list back to this node
                    nodeList->getNodeSocket()->send((sockaddr*)&replyDestinationSocket,
                                                    broadcastPacket,
                                                    (currentBufferPos - startPointer) + numHeaderBytes);
                }
            } else if (packetData[0] == PACKET_TYPE_REQUEST_ASSIGNMENT) {
                
                qDebug("Received a request for assignment.\n");
                
                ::assignmentQueueMutex.lock();
                
                // this is an unassigned client talking to us directly for an assignment
                // go through our queue and see if there are any assignments to give out
                std::deque<Assignment*>::iterator assignment = ::assignmentQueue.begin();
                
                while (assignment != ::assignmentQueue.end()) {
                    
                    // give this assignment out, no conditions stop us from giving it to the local assignment client
                    int numHeaderBytes = populateTypeAndVersion(broadcastPacket, PACKET_TYPE_CREATE_ASSIGNMENT);
                    int numAssignmentBytes = (*assignment)->packToBuffer(broadcastPacket + numHeaderBytes);
                    
                    nodeList->getNodeSocket()->send((sockaddr*) &nodePublicAddress,
                                                    broadcastPacket,
                                                    numHeaderBytes + numAssignmentBytes);
                    
                    // remove the assignment from the queue
                    ::assignmentQueue.erase(assignment);
                    
                    if ((*assignment)->getType() == Assignment::AgentType) {
                        // if this is a script assignment we need to delete it to avoid a memory leak
                        delete *assignment;
                    }
                    
                    // stop looping, we've handed out an assignment
                    break;
                }
                
                ::assignmentQueueMutex.unlock();
            }
        }
        
        // if ASSIGNMENT_REQUEST_INTERVAL_USECS have passed since last global assignment request then fire off another
        if (usecTimestampNow() - usecTimestamp(&lastGlobalAssignmentRequest) >= GLOBAL_ASSIGNMENT_REQUEST_INTERVAL_USECS) {
            gettimeofday(&lastGlobalAssignmentRequest, NULL);
            
            ::assignmentQueueMutex.lock();
            
            // go through our queue and see if there are any assignments to send to the global assignment server
            std::deque<Assignment*>::iterator assignment = ::assignmentQueue.begin();
            
            while (assignment != assignmentQueue.end()) {
                
                if ((*assignment)->getLocation() != Assignment::LocalLocation) {                    
                    // attach our local socket to the assignment so the assignment-server can optionally hand it out
                    (*assignment)->setAttachedLocalSocket((sockaddr*) &localSocket);
                    
                    nodeList->sendAssignment(*(*assignment));
                    
                    // remove the assignment from the queue
                    ::assignmentQueue.erase(assignment);
                    
                    if ((*assignment)->getType() == Assignment::AgentType) {
                        // if this is a script assignment we need to delete it to avoid a memory leak
                        delete *assignment;
                    }
                    
                    // stop looping, we've handed out an assignment
                    break;
                } else {
                    // push forward the iterator to check the next assignment
                    assignment++;
                }
            }
            
            ::assignmentQueueMutex.unlock();
        }
        
        if (Logging::shouldSendStats()) {
            if (usecTimestampNow() - usecTimestamp(&lastStatSendTime) >= (NODE_COUNT_STAT_INTERVAL_MSECS * 1000)) {
                // time to send our count of nodes and servers to logstash
                const char NODE_COUNT_LOGSTASH_KEY[] = "ds-node-count";
                
                Logging::stashValue(STAT_TYPE_TIMER, NODE_COUNT_LOGSTASH_KEY, nodeList->getNumAliveNodes());
                
                gettimeofday(&lastStatSendTime, NULL);
            }
        }
    }

    return 0;
}
Пример #4
0
int main(int argc, const char * argv[])
{
    NodeList* nodeList = NodeList::createInstance(NODE_TYPE_DOMAIN, DOMAIN_LISTEN_PORT);
	// If user asks to run in "local" mode then we do NOT replace the IP
	// with the EC2 IP. Otherwise, we will replace the IP like we used to
	// this allows developers to run a local domain without recompiling the
	// domain server
	bool isLocalMode = cmdOptionExists(argc, argv, "--local");
	if (isLocalMode) {
		printf("NOTE: Running in Local Mode!\n");
	} else {
		printf("--------------------------------------------------\n");
		printf("NOTE: Running in EC2 Mode. \n");
		printf("If you're a developer testing a local system, you\n");
		printf("probably want to include --local on command line.\n");
		printf("--------------------------------------------------\n");
	}

    setvbuf(stdout, NULL, _IOLBF, 0);
    
    ssize_t receivedBytes = 0;
    char nodeType = '\0';
    
    unsigned char broadcastPacket[MAX_PACKET_SIZE];
    int numHeaderBytes = populateTypeAndVersion(broadcastPacket, PACKET_TYPE_DOMAIN);
    
    unsigned char* currentBufferPos;
    unsigned char* startPointer;
    
    sockaddr_in nodePublicAddress, nodeLocalAddress;
    nodeLocalAddress.sin_family = AF_INET;
    
    in_addr_t serverLocalAddress = getLocalAddress();
    
    nodeList->startSilentNodeRemovalThread();
    
    timeval lastStatSendTime = {};
    
    while (true) {
        if (nodeList->getNodeSocket()->receive((sockaddr *)&nodePublicAddress, packetData, &receivedBytes) &&
            (packetData[0] == PACKET_TYPE_DOMAIN_REPORT_FOR_DUTY || packetData[0] == PACKET_TYPE_DOMAIN_LIST_REQUEST) &&
            packetVersionMatch(packetData)) {
            // this is an RFD or domain list request packet, and there is a version match
            std::map<char, Node *> newestSoloNodes;
            
            int numBytesSenderHeader = numBytesForPacketHeader(packetData);
            
            nodeType = *(packetData + numBytesSenderHeader);
            int numBytesSocket = unpackSocket(packetData + numBytesSenderHeader + sizeof(NODE_TYPE),
                                              (sockaddr*) &nodeLocalAddress);
            
            sockaddr* destinationSocket = (sockaddr*) &nodePublicAddress;
            
            // check the node public address
            // if it matches our local address we're on the same box
            // so hardcode the EC2 public address for now
            if (nodePublicAddress.sin_addr.s_addr == serverLocalAddress) {
            	// If we're not running "local" then we do replace the IP
            	// with the EC2 IP. Otherwise, we use our normal public IP
            	if (!isLocalMode) {
	                nodePublicAddress.sin_addr.s_addr = 895283510; // local IP in this format...
                    destinationSocket = (sockaddr*) &nodeLocalAddress;
	            }
            }
            
            Node* newNode = nodeList->addOrUpdateNode((sockaddr*) &nodePublicAddress,
                                                      (sockaddr*) &nodeLocalAddress,
                                                      nodeType,
                                                      nodeList->getLastNodeID());
            
            if (newNode->getNodeID() == nodeList->getLastNodeID()) {
                nodeList->increaseNodeID();
            }
            
            currentBufferPos = broadcastPacket + numHeaderBytes;
            startPointer = currentBufferPos;
            
            unsigned char* nodeTypesOfInterest = packetData + numBytesSenderHeader + sizeof(NODE_TYPE)
                + numBytesSocket + sizeof(unsigned char);
            int numInterestTypes = *(nodeTypesOfInterest - 1);
            
            if (numInterestTypes > 0) {
                // if the node has sent no types of interest, assume they want nothing but their own ID back
                for (NodeList::iterator node = nodeList->begin(); node != nodeList->end(); node++) {
                    if (!node->matches((sockaddr*) &nodePublicAddress, (sockaddr*) &nodeLocalAddress, nodeType) &&
                            memchr(nodeTypesOfInterest, node->getType(), numInterestTypes)) {
                        // this is not the node themselves
                        // and this is an node of a type in the passed node types of interest
                        // or the node did not pass us any specific types they are interested in
                    
                        if (memchr(SOLO_NODE_TYPES, node->getType(), sizeof(SOLO_NODE_TYPES)) == NULL) {
                            // this is an node of which there can be multiple, just add them to the packet
                            // don't send avatar nodes to other avatars, that will come from avatar mixer
                            if (nodeType != NODE_TYPE_AGENT || node->getType() != NODE_TYPE_AGENT) {
                                currentBufferPos = addNodeToBroadcastPacket(currentBufferPos, &(*node));
                            }
                        
                        } else {
                            // solo node, we need to only send newest
                            if (newestSoloNodes[node->getType()] == NULL ||
                                newestSoloNodes[node->getType()]->getWakeMicrostamp() < node->getWakeMicrostamp()) {
                                // we have to set the newer solo node to add it to the broadcast later
                                newestSoloNodes[node->getType()] = &(*node);
                            }
                        }
                    }
                }
                
                for (std::map<char, Node *>::iterator soloNode = newestSoloNodes.begin();
                     soloNode != newestSoloNodes.end();
                     soloNode++) {
                    // this is the newest alive solo node, add them to the packet
                    currentBufferPos = addNodeToBroadcastPacket(currentBufferPos, soloNode->second);
                }
            }
                        
            // update last receive to now
            uint64_t timeNow = usecTimestampNow();
            newNode->setLastHeardMicrostamp(timeNow);
            
            if (packetData[0] == PACKET_TYPE_DOMAIN_REPORT_FOR_DUTY
                && memchr(SOLO_NODE_TYPES, nodeType, sizeof(SOLO_NODE_TYPES))) {
                newNode->setWakeMicrostamp(timeNow);
            }
            
            // add the node ID to the end of the pointer
            currentBufferPos += packNodeId(currentBufferPos, newNode->getNodeID());
            
            // send the constructed list back to this node
            nodeList->getNodeSocket()->send(destinationSocket,
                                            broadcastPacket,
                                            (currentBufferPos - startPointer) + numHeaderBytes);
        }
        
        if (Logstash::shouldSendStats()) {
            if (usecTimestampNow() - usecTimestamp(&lastStatSendTime) >= (NODE_COUNT_STAT_INTERVAL_MSECS * 1000)) {
                // time to send our count of nodes and servers to logstash
                const char NODE_COUNT_LOGSTASH_KEY[] = "ds-node-count";
                
                Logstash::stashValue(STAT_TYPE_TIMER, NODE_COUNT_LOGSTASH_KEY, nodeList->getNumAliveNodes());
                
                gettimeofday(&lastStatSendTime, NULL);
            }
        }
    }

    return 0;
}