int AgentList::processDomainServerList(unsigned char *packetData, size_t dataBytes) {
int readAgents = 0;
char agentType;
uint16_t agentId;
// assumes only IPv4 addresses
sockaddr_in agentPublicSocket;
agentPublicSocket.sin_family = AF_INET;
sockaddr_in agentLocalSocket;
agentLocalSocket.sin_family = AF_INET;
unsigned char *readPtr = packetData + 1;
unsigned char *startPtr = packetData;
while((readPtr - startPtr) < dataBytes - sizeof(uint16_t)) {
agentType = *readPtr++;
readPtr += unpackAgentId(readPtr, (uint16_t *)&agentId);
readPtr += unpackSocket(readPtr, (sockaddr *)&agentPublicSocket);
readPtr += unpackSocket(readPtr, (sockaddr *)&agentLocalSocket);
addOrUpdateAgent((sockaddr *)&agentPublicSocket, (sockaddr *)&agentLocalSocket, agentType, agentId);
}
// read out our ID from the packet
unpackAgentId(readPtr, &_ownerID);
return readAgents;
}
Assignment::Assignment(const unsigned char* dataBuffer, int numBytes) :
_location(GlobalLocation),
_attachedPublicSocket(NULL),
_attachedLocalSocket(NULL)
{
// set the create time on this assignment
gettimeofday(&_time, NULL);
int numBytesRead = 0;
if (dataBuffer[0] == PACKET_TYPE_REQUEST_ASSIGNMENT) {
_command = Assignment::RequestCommand;
} else if (dataBuffer[0] == PACKET_TYPE_CREATE_ASSIGNMENT) {
_command = Assignment::CreateCommand;
} else if (dataBuffer[0] == PACKET_TYPE_DEPLOY_ASSIGNMENT) {
_command = Assignment::DeployCommand;
}
numBytesRead += numBytesForPacketHeader(dataBuffer);
if (dataBuffer[0] != PACKET_TYPE_REQUEST_ASSIGNMENT) {
// read the GUID for this assignment
_uuid = QUuid::fromRfc4122(QByteArray((const char*) dataBuffer + numBytesRead, NUM_BYTES_RFC4122_UUID));
numBytesRead += NUM_BYTES_RFC4122_UUID;
}
memcpy(&_type, dataBuffer + numBytesRead, sizeof(Assignment::Type));
numBytesRead += sizeof(Assignment::Type);
if (numBytes > numBytesRead) {
sockaddr* newSocket = NULL;
if (dataBuffer[numBytesRead++] == IPv4_ADDRESS_DESIGNATOR) {
// IPv4 address
newSocket = (sockaddr*) new sockaddr_in;
unpackSocket(dataBuffer + numBytesRead, newSocket);
} else {
// IPv6 address, or bad designator
qDebug("Received a socket that cannot be unpacked!\n");
}
if (_command == Assignment::CreateCommand) {
delete _attachedLocalSocket;
_attachedLocalSocket = newSocket;
} else {
delete _attachedPublicSocket;
_attachedPublicSocket = newSocket;
}
}
}
int NodeList::processDomainServerList(unsigned char* packetData, size_t dataBytes) {
// this is a packet from the domain server, reset the count of un-replied check-ins
_numNoReplyDomainCheckIns = 0;
int readNodes = 0;
char nodeType;
uint16_t nodeId;
// assumes only IPv4 addresses
sockaddr_in nodePublicSocket;
nodePublicSocket.sin_family = AF_INET;
sockaddr_in nodeLocalSocket;
nodeLocalSocket.sin_family = AF_INET;
unsigned char* readPtr = packetData + numBytesForPacketHeader(packetData);
unsigned char* startPtr = packetData;
while((readPtr - startPtr) < dataBytes - sizeof(uint16_t)) {
nodeType = *readPtr++;
readPtr += unpackNodeId(readPtr, (uint16_t*) &nodeId);
readPtr += unpackSocket(readPtr, (sockaddr*) &nodePublicSocket);
readPtr += unpackSocket(readPtr, (sockaddr*) &nodeLocalSocket);
// if the public socket address is 0 then it's reachable at the same IP
// as the domain server
if (nodePublicSocket.sin_addr.s_addr == 0) {
nodePublicSocket.sin_addr.s_addr = htonl(_domainIP.toIPv4Address());
}
addOrUpdateNode((sockaddr*) &nodePublicSocket, (sockaddr*) &nodeLocalSocket, nodeType, nodeId);
}
// read out our ID from the packet
unpackNodeId(readPtr, &_ownerID);
return readNodes;
}
int DomainServer::run() {
NodeList* nodeList = NodeList::getInstance();
nodeList->addHook(this);
ssize_t receivedBytes = 0;
char nodeType = '\0';
unsigned char broadcastPacket[MAX_PACKET_SIZE];
unsigned char packetData[MAX_PACKET_SIZE];
unsigned char* currentBufferPos;
unsigned char* startPointer;
sockaddr_in senderAddress, nodePublicAddress, nodeLocalAddress;
nodePublicAddress.sin_family = AF_INET;
nodeLocalAddress.sin_family = AF_INET;
nodeList->startSilentNodeRemovalThread();
if (!_staticAssignmentFile.exists() || _voxelServerConfig) {
if (_voxelServerConfig) {
// we have a new VS config, clear the existing file to start fresh
_staticAssignmentFile.remove();
}
prepopulateStaticAssignmentFile();
}
_staticAssignmentFile.open(QIODevice::ReadWrite);
_staticAssignmentFileData = _staticAssignmentFile.map(0, _staticAssignmentFile.size());
_staticAssignments = (Assignment*) _staticAssignmentFileData;
timeval startTime;
gettimeofday(&startTime, NULL);
while (true) {
while (nodeList->getNodeSocket()->receive((sockaddr *)&senderAddress, packetData, &receivedBytes) &&
packetVersionMatch(packetData)) {
if (packetData[0] == PACKET_TYPE_DOMAIN_REPORT_FOR_DUTY || packetData[0] == PACKET_TYPE_DOMAIN_LIST_REQUEST) {
// this is an RFD or domain list request packet, and there is a version match
int numBytesSenderHeader = numBytesForPacketHeader(packetData);
nodeType = *(packetData + numBytesSenderHeader);
int packetIndex = numBytesSenderHeader + sizeof(NODE_TYPE);
QUuid nodeUUID = QUuid::fromRfc4122(QByteArray(((char*) packetData + packetIndex), NUM_BYTES_RFC4122_UUID));
packetIndex += NUM_BYTES_RFC4122_UUID;
int numBytesPrivateSocket = unpackSocket(packetData + packetIndex, (sockaddr*) &nodePublicAddress);
packetIndex += numBytesPrivateSocket;
if (nodePublicAddress.sin_addr.s_addr == 0) {
// this node wants to use us its STUN server
// so set the node public address to whatever we perceive the public address to be
nodePublicAddress = senderAddress;
// if the sender is on our box then leave its public address to 0 so that
// other users attempt to reach it on the same address they have for the domain-server
if (senderAddress.sin_addr.s_addr == htonl(INADDR_LOOPBACK)) {
nodePublicAddress.sin_addr.s_addr = 0;
}
}
int numBytesPublicSocket = unpackSocket(packetData + packetIndex, (sockaddr*) &nodeLocalAddress);
packetIndex += numBytesPublicSocket;
const char STATICALLY_ASSIGNED_NODES[3] = {
NODE_TYPE_AUDIO_MIXER,
NODE_TYPE_AVATAR_MIXER,
NODE_TYPE_VOXEL_SERVER
};
Assignment* matchingStaticAssignment = NULL;
if (memchr(STATICALLY_ASSIGNED_NODES, nodeType, sizeof(STATICALLY_ASSIGNED_NODES)) == NULL
|| ((matchingStaticAssignment = matchingStaticAssignmentForCheckIn(nodeUUID, nodeType))
|| checkInWithUUIDMatchesExistingNode((sockaddr*) &nodePublicAddress,
(sockaddr*) &nodeLocalAddress,
nodeUUID)))
{
Node* checkInNode = nodeList->addOrUpdateNode(nodeUUID,
nodeType,
(sockaddr*) &nodePublicAddress,
(sockaddr*) &nodeLocalAddress);
if (matchingStaticAssignment) {
// this was a newly added node with a matching static assignment
if (_hasCompletedRestartHold) {
// remove the matching assignment from the assignment queue so we don't take the next check in
removeAssignmentFromQueue(matchingStaticAssignment);
}
// set the linked data for this node to a copy of the matching assignment
// so we can re-queue it should the node die
Assignment* nodeCopyOfMatchingAssignment = new Assignment(*matchingStaticAssignment);
checkInNode->setLinkedData(nodeCopyOfMatchingAssignment);
}
int numHeaderBytes = populateTypeAndVersion(broadcastPacket, PACKET_TYPE_DOMAIN);
currentBufferPos = broadcastPacket + numHeaderBytes;
startPointer = currentBufferPos;
unsigned char* nodeTypesOfInterest = packetData + packetIndex + sizeof(unsigned char);
int numInterestTypes = *(nodeTypesOfInterest - 1);
if (numInterestTypes > 0) {
// if the node has sent no types of interest, assume they want nothing but their own ID back
for (NodeList::iterator node = nodeList->begin(); node != nodeList->end(); node++) {
if (!node->matches((sockaddr*) &nodePublicAddress, (sockaddr*) &nodeLocalAddress, nodeType) &&
memchr(nodeTypesOfInterest, node->getType(), numInterestTypes)) {
// don't send avatar nodes to other avatars, that will come from avatar mixer
if (nodeType != NODE_TYPE_AGENT || node->getType() != NODE_TYPE_AGENT) {
currentBufferPos = addNodeToBroadcastPacket(currentBufferPos, &(*node));
}
}
}
}
// update last receive to now
uint64_t timeNow = usecTimestampNow();
checkInNode->setLastHeardMicrostamp(timeNow);
// send the constructed list back to this node
nodeList->getNodeSocket()->send((sockaddr*)&senderAddress,
broadcastPacket,
(currentBufferPos - startPointer) + numHeaderBytes);
}
} else if (packetData[0] == PACKET_TYPE_REQUEST_ASSIGNMENT) {
qDebug("Received a request for assignment.\n");
if (!_hasCompletedRestartHold) {
possiblyAddStaticAssignmentsBackToQueueAfterRestart(&startTime);
}
if (_assignmentQueue.size() > 0) {
// construct the requested assignment from the packet data
Assignment requestAssignment(packetData, receivedBytes);
Assignment* assignmentToDeploy = deployableAssignmentForRequest(requestAssignment);
if (assignmentToDeploy) {
// give this assignment out, either the type matches or the requestor said they will take any
int numHeaderBytes = populateTypeAndVersion(broadcastPacket, PACKET_TYPE_CREATE_ASSIGNMENT);
int numAssignmentBytes = assignmentToDeploy->packToBuffer(broadcastPacket + numHeaderBytes);
nodeList->getNodeSocket()->send((sockaddr*) &senderAddress,
broadcastPacket,
numHeaderBytes + numAssignmentBytes);
if (assignmentToDeploy->getNumberOfInstances() == 0) {
// there are no more instances of this script to send out, delete it
delete assignmentToDeploy;
}
}
}
} else if (packetData[0] == PACKET_TYPE_CREATE_ASSIGNMENT) {
// this is a create assignment likely recieved from a server needed more clients to help with load
// unpack it
Assignment* createAssignment = new Assignment(packetData, receivedBytes);
qDebug() << "Received a create assignment -" << *createAssignment << "\n";
// make sure we have a matching node with the UUID packed with the assignment
// if the node has sent no types of interest, assume they want nothing but their own ID back
for (NodeList::iterator node = nodeList->begin(); node != nodeList->end(); node++) {
if (node->getLinkedData()
&& socketMatch((sockaddr*) &senderAddress, node->getPublicSocket())
&& ((Assignment*) node->getLinkedData())->getUUID() == createAssignment->getUUID()) {
// give the create assignment a new UUID
createAssignment->resetUUID();
// add the assignment at the back of the queue
_assignmentQueueMutex.lock();
_assignmentQueue.push_back(createAssignment);
_assignmentQueueMutex.unlock();
// find the first available spot in the static assignments and put this assignment there
for (int i = 0; i < MAX_STATIC_ASSIGNMENT_FILE_ASSIGNMENTS; i++) {
if (_staticAssignments[i].getUUID().isNull()) {
_staticAssignments[i] = *createAssignment;
// we've stuck the assignment in, break out
break;
}
}
// we found the matching node that asked for create assignment, break out
break;
}
}
}
}
if (!_hasCompletedRestartHold) {
possiblyAddStaticAssignmentsBackToQueueAfterRestart(&startTime);
}
}
this->cleanup();
return 0;
}
int main(int argc, const char* argv[]) {
qInstallMessageHandler(Logging::verboseMessageHandler);
NodeList* nodeList = NodeList::createInstance(NODE_TYPE_DOMAIN, DOMAIN_LISTEN_PORT);
setvbuf(stdout, NULL, _IOLBF, 0);
ssize_t receivedBytes = 0;
char nodeType = '\0';
unsigned char broadcastPacket[MAX_PACKET_SIZE];
unsigned char* currentBufferPos;
unsigned char* startPointer;
sockaddr_in nodePublicAddress, nodeLocalAddress, replyDestinationSocket;
nodeLocalAddress.sin_family = AF_INET;
in_addr_t serverLocalAddress = getLocalAddress();
nodeList->startSilentNodeRemovalThread();
timeval lastStatSendTime = {};
const char ASSIGNMENT_SERVER_OPTION[] = "-a";
// grab the overriden assignment-server hostname from argv, if it exists
const char* customAssignmentServer = getCmdOption(argc, argv, ASSIGNMENT_SERVER_OPTION);
if (customAssignmentServer) {
sockaddr_in customAssignmentSocket = socketForHostnameAndHostOrderPort(customAssignmentServer, ASSIGNMENT_SERVER_PORT);
nodeList->setAssignmentServerSocket((sockaddr*) &customAssignmentSocket);
}
// use a map to keep track of iterations of silence for assignment creation requests
const long long GLOBAL_ASSIGNMENT_REQUEST_INTERVAL_USECS = 1 * 1000 * 1000;
timeval lastGlobalAssignmentRequest = {};
// as a domain-server we will always want an audio mixer and avatar mixer
// setup the create assignments for those
Assignment audioMixerAssignment(Assignment::CreateCommand,
Assignment::AudioMixerType,
Assignment::LocalLocation);
Assignment avatarMixerAssignment(Assignment::CreateCommand,
Assignment::AvatarMixerType,
Assignment::LocalLocation);
// construct a local socket to send with our created assignments to the global AS
sockaddr_in localSocket = {};
localSocket.sin_family = AF_INET;
localSocket.sin_port = htons(nodeList->getInstance()->getNodeSocket()->getListeningPort());
localSocket.sin_addr.s_addr = serverLocalAddress;
// setup the mongoose web server
struct mg_context *ctx;
struct mg_callbacks callbacks = {};
// list of options. Last element must be NULL.
const char *options[] = {"listening_ports", "8080",
"document_root", "./resources/web", NULL};
callbacks.begin_request = mongooseRequestHandler;
callbacks.upload = mongooseUploadHandler;
// Start the web server.
ctx = mg_start(&callbacks, NULL, options);
while (true) {
::assignmentQueueMutex.lock();
// check if our audio-mixer or avatar-mixer are dead and we don't have existing assignments in the queue
// so we can add those assignments back to the front of the queue since they are high-priority
if (!nodeList->soloNodeOfType(NODE_TYPE_AVATAR_MIXER) &&
std::find(::assignmentQueue.begin(), assignmentQueue.end(), &avatarMixerAssignment) == ::assignmentQueue.end()) {
qDebug("Missing an avatar mixer and assignment not in queue. Adding.\n");
::assignmentQueue.push_front(&avatarMixerAssignment);
}
if (!nodeList->soloNodeOfType(NODE_TYPE_AUDIO_MIXER) &&
std::find(::assignmentQueue.begin(), ::assignmentQueue.end(), &audioMixerAssignment) == ::assignmentQueue.end()) {
qDebug("Missing an audio mixer and assignment not in queue. Adding.\n");
::assignmentQueue.push_front(&audioMixerAssignment);
}
::assignmentQueueMutex.unlock();
while (nodeList->getNodeSocket()->receive((sockaddr *)&nodePublicAddress, packetData, &receivedBytes) &&
packetVersionMatch(packetData)) {
if (packetData[0] == PACKET_TYPE_DOMAIN_REPORT_FOR_DUTY || packetData[0] == PACKET_TYPE_DOMAIN_LIST_REQUEST) {
// this is an RFD or domain list request packet, and there is a version match
std::map<char, Node *> newestSoloNodes;
int numBytesSenderHeader = numBytesForPacketHeader(packetData);
nodeType = *(packetData + numBytesSenderHeader);
int numBytesSocket = unpackSocket(packetData + numBytesSenderHeader + sizeof(NODE_TYPE),
(sockaddr*) &nodeLocalAddress);
replyDestinationSocket = nodePublicAddress;
// check the node public address
// if it matches our local address
// or if it's the loopback address we're on the same box
if (nodePublicAddress.sin_addr.s_addr == serverLocalAddress ||
nodePublicAddress.sin_addr.s_addr == htonl(INADDR_LOOPBACK)) {
nodePublicAddress.sin_addr.s_addr = 0;
}
Node* newNode = nodeList->addOrUpdateNode((sockaddr*) &nodePublicAddress,
(sockaddr*) &nodeLocalAddress,
nodeType,
nodeList->getLastNodeID());
// if addOrUpdateNode returns NULL this was a solo node we already have, don't talk back to it
if (newNode) {
if (newNode->getNodeID() == nodeList->getLastNodeID()) {
nodeList->increaseNodeID();
}
int numHeaderBytes = populateTypeAndVersion(broadcastPacket, PACKET_TYPE_DOMAIN);
currentBufferPos = broadcastPacket + numHeaderBytes;
startPointer = currentBufferPos;
unsigned char* nodeTypesOfInterest = packetData + numBytesSenderHeader + sizeof(NODE_TYPE)
+ numBytesSocket + sizeof(unsigned char);
int numInterestTypes = *(nodeTypesOfInterest - 1);
if (numInterestTypes > 0) {
// if the node has sent no types of interest, assume they want nothing but their own ID back
for (NodeList::iterator node = nodeList->begin(); node != nodeList->end(); node++) {
if (!node->matches((sockaddr*) &nodePublicAddress, (sockaddr*) &nodeLocalAddress, nodeType) &&
memchr(nodeTypesOfInterest, node->getType(), numInterestTypes)) {
// this is not the node themselves
// and this is an node of a type in the passed node types of interest
// or the node did not pass us any specific types they are interested in
if (memchr(SOLO_NODE_TYPES, node->getType(), sizeof(SOLO_NODE_TYPES)) == NULL) {
// this is an node of which there can be multiple, just add them to the packet
// don't send avatar nodes to other avatars, that will come from avatar mixer
if (nodeType != NODE_TYPE_AGENT || node->getType() != NODE_TYPE_AGENT) {
currentBufferPos = addNodeToBroadcastPacket(currentBufferPos, &(*node));
}
} else {
// solo node, we need to only send newest
if (newestSoloNodes[node->getType()] == NULL ||
newestSoloNodes[node->getType()]->getWakeMicrostamp() < node->getWakeMicrostamp()) {
// we have to set the newer solo node to add it to the broadcast later
newestSoloNodes[node->getType()] = &(*node);
}
}
}
}
for (std::map<char, Node *>::iterator soloNode = newestSoloNodes.begin();
soloNode != newestSoloNodes.end();
soloNode++) {
// this is the newest alive solo node, add them to the packet
currentBufferPos = addNodeToBroadcastPacket(currentBufferPos, soloNode->second);
}
}
// update last receive to now
uint64_t timeNow = usecTimestampNow();
newNode->setLastHeardMicrostamp(timeNow);
if (packetData[0] == PACKET_TYPE_DOMAIN_REPORT_FOR_DUTY
&& memchr(SOLO_NODE_TYPES, nodeType, sizeof(SOLO_NODE_TYPES))) {
newNode->setWakeMicrostamp(timeNow);
}
// add the node ID to the end of the pointer
currentBufferPos += packNodeId(currentBufferPos, newNode->getNodeID());
// send the constructed list back to this node
nodeList->getNodeSocket()->send((sockaddr*)&replyDestinationSocket,
broadcastPacket,
(currentBufferPos - startPointer) + numHeaderBytes);
}
} else if (packetData[0] == PACKET_TYPE_REQUEST_ASSIGNMENT) {
qDebug("Received a request for assignment.\n");
::assignmentQueueMutex.lock();
// this is an unassigned client talking to us directly for an assignment
// go through our queue and see if there are any assignments to give out
std::deque<Assignment*>::iterator assignment = ::assignmentQueue.begin();
while (assignment != ::assignmentQueue.end()) {
// give this assignment out, no conditions stop us from giving it to the local assignment client
int numHeaderBytes = populateTypeAndVersion(broadcastPacket, PACKET_TYPE_CREATE_ASSIGNMENT);
int numAssignmentBytes = (*assignment)->packToBuffer(broadcastPacket + numHeaderBytes);
nodeList->getNodeSocket()->send((sockaddr*) &nodePublicAddress,
broadcastPacket,
numHeaderBytes + numAssignmentBytes);
// remove the assignment from the queue
::assignmentQueue.erase(assignment);
if ((*assignment)->getType() == Assignment::AgentType) {
// if this is a script assignment we need to delete it to avoid a memory leak
delete *assignment;
}
// stop looping, we've handed out an assignment
break;
}
::assignmentQueueMutex.unlock();
}
}
// if ASSIGNMENT_REQUEST_INTERVAL_USECS have passed since last global assignment request then fire off another
if (usecTimestampNow() - usecTimestamp(&lastGlobalAssignmentRequest) >= GLOBAL_ASSIGNMENT_REQUEST_INTERVAL_USECS) {
gettimeofday(&lastGlobalAssignmentRequest, NULL);
::assignmentQueueMutex.lock();
// go through our queue and see if there are any assignments to send to the global assignment server
std::deque<Assignment*>::iterator assignment = ::assignmentQueue.begin();
while (assignment != assignmentQueue.end()) {
if ((*assignment)->getLocation() != Assignment::LocalLocation) {
// attach our local socket to the assignment so the assignment-server can optionally hand it out
(*assignment)->setAttachedLocalSocket((sockaddr*) &localSocket);
nodeList->sendAssignment(*(*assignment));
// remove the assignment from the queue
::assignmentQueue.erase(assignment);
if ((*assignment)->getType() == Assignment::AgentType) {
// if this is a script assignment we need to delete it to avoid a memory leak
delete *assignment;
}
// stop looping, we've handed out an assignment
break;
} else {
// push forward the iterator to check the next assignment
assignment++;
}
}
::assignmentQueueMutex.unlock();
}
if (Logging::shouldSendStats()) {
if (usecTimestampNow() - usecTimestamp(&lastStatSendTime) >= (NODE_COUNT_STAT_INTERVAL_MSECS * 1000)) {
// time to send our count of nodes and servers to logstash
const char NODE_COUNT_LOGSTASH_KEY[] = "ds-node-count";
Logging::stashValue(STAT_TYPE_TIMER, NODE_COUNT_LOGSTASH_KEY, nodeList->getNumAliveNodes());
gettimeofday(&lastStatSendTime, NULL);
}
}
}
return 0;
}
int main(int argc, const char * argv[])
{
// If user asks to run in "local" mode then we do NOT replace the IP
// with the EC2 IP. Otherwise, we will replace the IP like we used to
// this allows developers to run a local domain without recompiling the
// domain server
bool useLocal = cmdOptionExists(argc, argv, "--local");
if (useLocal) {
printf("NOTE: Running in Local Mode!\n");
} else {
printf("--------------------------------------------------\n");
printf("NOTE: Running in EC2 Mode. \n");
printf("If you're a developer testing a local system, you\n");
printf("probably want to include --local on command line.\n");
printf("--------------------------------------------------\n");
}
setvbuf(stdout, NULL, _IOLBF, 0);
ssize_t receivedBytes = 0;
char agentType;
unsigned char *broadcastPacket = new unsigned char[MAX_PACKET_SIZE];
*broadcastPacket = 'D';
unsigned char *currentBufferPos;
unsigned char *startPointer;
int packetBytesWithoutLeadingChar;
sockaddr_in agentPublicAddress, agentLocalAddress;
agentLocalAddress.sin_family = AF_INET;
in_addr_t serverLocalAddress = getLocalAddress();
agentList.startSilentAgentRemovalThread();
while (true) {
if (agentList.getAgentSocket().receive((sockaddr *)&agentPublicAddress, packetData, &receivedBytes)) {
std::map<char, Agent *> newestSoloAgents;
agentType = packetData[0];
unpackSocket(&packetData[1], (sockaddr *)&agentLocalAddress);
// check the agent public address
// if it matches our local address we're on the same box
// so hardcode the EC2 public address for now
if (agentPublicAddress.sin_addr.s_addr == serverLocalAddress) {
// If we're not running "local" then we do replace the IP
// with the EC2 IP. Otherwise, we use our normal public IP
if (!useLocal) {
agentPublicAddress.sin_addr.s_addr = 895283510; // local IP in this format...
}
}
if (agentList.addOrUpdateAgent((sockaddr *)&agentPublicAddress,
(sockaddr *)&agentLocalAddress,
agentType,
agentList.getLastAgentId())) {
agentList.increaseAgentId();
}
currentBufferPos = broadcastPacket + 1;
startPointer = currentBufferPos;
for(std::vector<Agent>::iterator agent = agentList.getAgents().begin(); agent != agentList.getAgents().end(); agent++) {
if (DEBUG_TO_SELF || !agent->matches((sockaddr *)&agentPublicAddress, (sockaddr *)&agentLocalAddress, agentType)) {
if (strchr(SOLO_AGENT_TYPES_STRING, (int) agent->getType()) == NULL) {
// this is an agent of which there can be multiple, just add them to the packet
currentBufferPos = addAgentToBroadcastPacket(currentBufferPos, &(*agent));
} else {
// solo agent, we need to only send newest
if (newestSoloAgents[agent->getType()] == NULL ||
newestSoloAgents[agent->getType()]->getFirstRecvTimeUsecs() < agent->getFirstRecvTimeUsecs()) {
// we have to set the newer solo agent to add it to the broadcast later
newestSoloAgents[agent->getType()] = &(*agent);
}
}
} else {
// this is the agent, just update last receive to now
agent->setLastRecvTimeUsecs(usecTimestampNow());
}
}
for (std::map<char, Agent *>::iterator agentIterator = newestSoloAgents.begin();
agentIterator != newestSoloAgents.end();
agentIterator++) {
// this is the newest alive solo agent, add them to the packet
currentBufferPos = addAgentToBroadcastPacket(currentBufferPos, agentIterator->second);
}
if ((packetBytesWithoutLeadingChar = (currentBufferPos - startPointer))) {
agentList.getAgentSocket().send((sockaddr *)&agentPublicAddress, broadcastPacket, packetBytesWithoutLeadingChar + 1);
}
}
}
return 0;
}
int main(int argc, const char * argv[])
{
NodeList* nodeList = NodeList::createInstance(NODE_TYPE_DOMAIN, DOMAIN_LISTEN_PORT);
// If user asks to run in "local" mode then we do NOT replace the IP
// with the EC2 IP. Otherwise, we will replace the IP like we used to
// this allows developers to run a local domain without recompiling the
// domain server
bool isLocalMode = cmdOptionExists(argc, argv, "--local");
if (isLocalMode) {
printf("NOTE: Running in Local Mode!\n");
} else {
printf("--------------------------------------------------\n");
printf("NOTE: Running in EC2 Mode. \n");
printf("If you're a developer testing a local system, you\n");
printf("probably want to include --local on command line.\n");
printf("--------------------------------------------------\n");
}
setvbuf(stdout, NULL, _IOLBF, 0);
ssize_t receivedBytes = 0;
char nodeType = '\0';
unsigned char broadcastPacket[MAX_PACKET_SIZE];
int numHeaderBytes = populateTypeAndVersion(broadcastPacket, PACKET_TYPE_DOMAIN);
unsigned char* currentBufferPos;
unsigned char* startPointer;
sockaddr_in nodePublicAddress, nodeLocalAddress;
nodeLocalAddress.sin_family = AF_INET;
in_addr_t serverLocalAddress = getLocalAddress();
nodeList->startSilentNodeRemovalThread();
timeval lastStatSendTime = {};
while (true) {
if (nodeList->getNodeSocket()->receive((sockaddr *)&nodePublicAddress, packetData, &receivedBytes) &&
(packetData[0] == PACKET_TYPE_DOMAIN_REPORT_FOR_DUTY || packetData[0] == PACKET_TYPE_DOMAIN_LIST_REQUEST) &&
packetVersionMatch(packetData)) {
// this is an RFD or domain list request packet, and there is a version match
std::map<char, Node *> newestSoloNodes;
int numBytesSenderHeader = numBytesForPacketHeader(packetData);
nodeType = *(packetData + numBytesSenderHeader);
int numBytesSocket = unpackSocket(packetData + numBytesSenderHeader + sizeof(NODE_TYPE),
(sockaddr*) &nodeLocalAddress);
sockaddr* destinationSocket = (sockaddr*) &nodePublicAddress;
// check the node public address
// if it matches our local address we're on the same box
// so hardcode the EC2 public address for now
if (nodePublicAddress.sin_addr.s_addr == serverLocalAddress) {
// If we're not running "local" then we do replace the IP
// with the EC2 IP. Otherwise, we use our normal public IP
if (!isLocalMode) {
nodePublicAddress.sin_addr.s_addr = 895283510; // local IP in this format...
destinationSocket = (sockaddr*) &nodeLocalAddress;
}
}
Node* newNode = nodeList->addOrUpdateNode((sockaddr*) &nodePublicAddress,
(sockaddr*) &nodeLocalAddress,
nodeType,
nodeList->getLastNodeID());
if (newNode->getNodeID() == nodeList->getLastNodeID()) {
nodeList->increaseNodeID();
}
currentBufferPos = broadcastPacket + numHeaderBytes;
startPointer = currentBufferPos;
unsigned char* nodeTypesOfInterest = packetData + numBytesSenderHeader + sizeof(NODE_TYPE)
+ numBytesSocket + sizeof(unsigned char);
int numInterestTypes = *(nodeTypesOfInterest - 1);
if (numInterestTypes > 0) {
// if the node has sent no types of interest, assume they want nothing but their own ID back
for (NodeList::iterator node = nodeList->begin(); node != nodeList->end(); node++) {
if (!node->matches((sockaddr*) &nodePublicAddress, (sockaddr*) &nodeLocalAddress, nodeType) &&
memchr(nodeTypesOfInterest, node->getType(), numInterestTypes)) {
// this is not the node themselves
// and this is an node of a type in the passed node types of interest
// or the node did not pass us any specific types they are interested in
if (memchr(SOLO_NODE_TYPES, node->getType(), sizeof(SOLO_NODE_TYPES)) == NULL) {
// this is an node of which there can be multiple, just add them to the packet
// don't send avatar nodes to other avatars, that will come from avatar mixer
if (nodeType != NODE_TYPE_AGENT || node->getType() != NODE_TYPE_AGENT) {
currentBufferPos = addNodeToBroadcastPacket(currentBufferPos, &(*node));
}
} else {
// solo node, we need to only send newest
if (newestSoloNodes[node->getType()] == NULL ||
newestSoloNodes[node->getType()]->getWakeMicrostamp() < node->getWakeMicrostamp()) {
// we have to set the newer solo node to add it to the broadcast later
newestSoloNodes[node->getType()] = &(*node);
}
}
}
}
for (std::map<char, Node *>::iterator soloNode = newestSoloNodes.begin();
soloNode != newestSoloNodes.end();
soloNode++) {
// this is the newest alive solo node, add them to the packet
currentBufferPos = addNodeToBroadcastPacket(currentBufferPos, soloNode->second);
}
}
// update last receive to now
uint64_t timeNow = usecTimestampNow();
newNode->setLastHeardMicrostamp(timeNow);
if (packetData[0] == PACKET_TYPE_DOMAIN_REPORT_FOR_DUTY
&& memchr(SOLO_NODE_TYPES, nodeType, sizeof(SOLO_NODE_TYPES))) {
newNode->setWakeMicrostamp(timeNow);
}
// add the node ID to the end of the pointer
currentBufferPos += packNodeId(currentBufferPos, newNode->getNodeID());
// send the constructed list back to this node
nodeList->getNodeSocket()->send(destinationSocket,
broadcastPacket,
(currentBufferPos - startPointer) + numHeaderBytes);
}
if (Logstash::shouldSendStats()) {
if (usecTimestampNow() - usecTimestamp(&lastStatSendTime) >= (NODE_COUNT_STAT_INTERVAL_MSECS * 1000)) {
// time to send our count of nodes and servers to logstash
const char NODE_COUNT_LOGSTASH_KEY[] = "ds-node-count";
Logstash::stashValue(STAT_TYPE_TIMER, NODE_COUNT_LOGSTASH_KEY, nodeList->getNumAliveNodes());
gettimeofday(&lastStatSendTime, NULL);
}
}
}
return 0;
}
