int DomainServer::run() {
NodeList* nodeList = NodeList::getInstance();
nodeList->addHook(this);
ssize_t receivedBytes = 0;
char nodeType = '\0';
unsigned char broadcastPacket[MAX_PACKET_SIZE];
unsigned char packetData[MAX_PACKET_SIZE];
unsigned char* currentBufferPos;
unsigned char* startPointer;
sockaddr_in senderAddress, nodePublicAddress, nodeLocalAddress;
nodePublicAddress.sin_family = AF_INET;
nodeLocalAddress.sin_family = AF_INET;
nodeList->startSilentNodeRemovalThread();
if (!_staticAssignmentFile.exists() || _voxelServerConfig) {
if (_voxelServerConfig) {
// we have a new VS config, clear the existing file to start fresh
_staticAssignmentFile.remove();
}
prepopulateStaticAssignmentFile();
}
_staticAssignmentFile.open(QIODevice::ReadWrite);
_staticAssignmentFileData = _staticAssignmentFile.map(0, _staticAssignmentFile.size());
_staticAssignments = (Assignment*) _staticAssignmentFileData;
timeval startTime;
gettimeofday(&startTime, NULL);
while (true) {
while (nodeList->getNodeSocket()->receive((sockaddr *)&senderAddress, packetData, &receivedBytes) &&
packetVersionMatch(packetData)) {
if (packetData[0] == PACKET_TYPE_DOMAIN_REPORT_FOR_DUTY || packetData[0] == PACKET_TYPE_DOMAIN_LIST_REQUEST) {
// this is an RFD or domain list request packet, and there is a version match
int numBytesSenderHeader = numBytesForPacketHeader(packetData);
nodeType = *(packetData + numBytesSenderHeader);
int packetIndex = numBytesSenderHeader + sizeof(NODE_TYPE);
QUuid nodeUUID = QUuid::fromRfc4122(QByteArray(((char*) packetData + packetIndex), NUM_BYTES_RFC4122_UUID));
packetIndex += NUM_BYTES_RFC4122_UUID;
int numBytesPrivateSocket = unpackSocket(packetData + packetIndex, (sockaddr*) &nodePublicAddress);
packetIndex += numBytesPrivateSocket;
if (nodePublicAddress.sin_addr.s_addr == 0) {
// this node wants to use us its STUN server
// so set the node public address to whatever we perceive the public address to be
nodePublicAddress = senderAddress;
// if the sender is on our box then leave its public address to 0 so that
// other users attempt to reach it on the same address they have for the domain-server
if (senderAddress.sin_addr.s_addr == htonl(INADDR_LOOPBACK)) {
nodePublicAddress.sin_addr.s_addr = 0;
}
}
int numBytesPublicSocket = unpackSocket(packetData + packetIndex, (sockaddr*) &nodeLocalAddress);
packetIndex += numBytesPublicSocket;
const char STATICALLY_ASSIGNED_NODES[3] = {
NODE_TYPE_AUDIO_MIXER,
NODE_TYPE_AVATAR_MIXER,
NODE_TYPE_VOXEL_SERVER
};
Assignment* matchingStaticAssignment = NULL;
if (memchr(STATICALLY_ASSIGNED_NODES, nodeType, sizeof(STATICALLY_ASSIGNED_NODES)) == NULL
|| ((matchingStaticAssignment = matchingStaticAssignmentForCheckIn(nodeUUID, nodeType))
|| checkInWithUUIDMatchesExistingNode((sockaddr*) &nodePublicAddress,
(sockaddr*) &nodeLocalAddress,
nodeUUID)))
{
Node* checkInNode = nodeList->addOrUpdateNode(nodeUUID,
nodeType,
(sockaddr*) &nodePublicAddress,
(sockaddr*) &nodeLocalAddress);
if (matchingStaticAssignment) {
// this was a newly added node with a matching static assignment
if (_hasCompletedRestartHold) {
// remove the matching assignment from the assignment queue so we don't take the next check in
removeAssignmentFromQueue(matchingStaticAssignment);
}
// set the linked data for this node to a copy of the matching assignment
// so we can re-queue it should the node die
Assignment* nodeCopyOfMatchingAssignment = new Assignment(*matchingStaticAssignment);
checkInNode->setLinkedData(nodeCopyOfMatchingAssignment);
}
int numHeaderBytes = populateTypeAndVersion(broadcastPacket, PACKET_TYPE_DOMAIN);
currentBufferPos = broadcastPacket + numHeaderBytes;
startPointer = currentBufferPos;
unsigned char* nodeTypesOfInterest = packetData + packetIndex + sizeof(unsigned char);
int numInterestTypes = *(nodeTypesOfInterest - 1);
if (numInterestTypes > 0) {
// if the node has sent no types of interest, assume they want nothing but their own ID back
for (NodeList::iterator node = nodeList->begin(); node != nodeList->end(); node++) {
if (!node->matches((sockaddr*) &nodePublicAddress, (sockaddr*) &nodeLocalAddress, nodeType) &&
memchr(nodeTypesOfInterest, node->getType(), numInterestTypes)) {
// don't send avatar nodes to other avatars, that will come from avatar mixer
if (nodeType != NODE_TYPE_AGENT || node->getType() != NODE_TYPE_AGENT) {
currentBufferPos = addNodeToBroadcastPacket(currentBufferPos, &(*node));
}
}
}
}
// update last receive to now
uint64_t timeNow = usecTimestampNow();
checkInNode->setLastHeardMicrostamp(timeNow);
// send the constructed list back to this node
nodeList->getNodeSocket()->send((sockaddr*)&senderAddress,
broadcastPacket,
(currentBufferPos - startPointer) + numHeaderBytes);
}
} else if (packetData[0] == PACKET_TYPE_REQUEST_ASSIGNMENT) {
qDebug("Received a request for assignment.\n");
if (!_hasCompletedRestartHold) {
possiblyAddStaticAssignmentsBackToQueueAfterRestart(&startTime);
}
if (_assignmentQueue.size() > 0) {
// construct the requested assignment from the packet data
Assignment requestAssignment(packetData, receivedBytes);
Assignment* assignmentToDeploy = deployableAssignmentForRequest(requestAssignment);
if (assignmentToDeploy) {
// give this assignment out, either the type matches or the requestor said they will take any
int numHeaderBytes = populateTypeAndVersion(broadcastPacket, PACKET_TYPE_CREATE_ASSIGNMENT);
int numAssignmentBytes = assignmentToDeploy->packToBuffer(broadcastPacket + numHeaderBytes);
nodeList->getNodeSocket()->send((sockaddr*) &senderAddress,
broadcastPacket,
numHeaderBytes + numAssignmentBytes);
if (assignmentToDeploy->getNumberOfInstances() == 0) {
// there are no more instances of this script to send out, delete it
delete assignmentToDeploy;
}
}
}
} else if (packetData[0] == PACKET_TYPE_CREATE_ASSIGNMENT) {
// this is a create assignment likely recieved from a server needed more clients to help with load
// unpack it
Assignment* createAssignment = new Assignment(packetData, receivedBytes);
qDebug() << "Received a create assignment -" << *createAssignment << "\n";
// make sure we have a matching node with the UUID packed with the assignment
// if the node has sent no types of interest, assume they want nothing but their own ID back
for (NodeList::iterator node = nodeList->begin(); node != nodeList->end(); node++) {
if (node->getLinkedData()
&& socketMatch((sockaddr*) &senderAddress, node->getPublicSocket())
&& ((Assignment*) node->getLinkedData())->getUUID() == createAssignment->getUUID()) {
// give the create assignment a new UUID
createAssignment->resetUUID();
// add the assignment at the back of the queue
_assignmentQueueMutex.lock();
_assignmentQueue.push_back(createAssignment);
_assignmentQueueMutex.unlock();
// find the first available spot in the static assignments and put this assignment there
for (int i = 0; i < MAX_STATIC_ASSIGNMENT_FILE_ASSIGNMENTS; i++) {
if (_staticAssignments[i].getUUID().isNull()) {
_staticAssignments[i] = *createAssignment;
// we've stuck the assignment in, break out
break;
}
}
// we found the matching node that asked for create assignment, break out
break;
}
}
}
}
if (!_hasCompletedRestartHold) {
possiblyAddStaticAssignmentsBackToQueueAfterRestart(&startTime);
}
}
this->cleanup();
return 0;
}
int main(int argc, const char* argv[]) {
NodeList* nodeList = NodeList::createInstance(NODE_TYPE_AVATAR_MIXER, AVATAR_LISTEN_PORT);
setvbuf(stdout, NULL, _IOLBF, 0);
// Handle Local Domain testing with the --local command line
const char* local = "--local";
if (cmdOptionExists(argc, argv, local)) {
printf("Local Domain MODE!\n");
nodeList->setDomainIPToLocalhost();
}
nodeList->linkedDataCreateCallback = attachAvatarDataToNode;
nodeList->startSilentNodeRemovalThread();
sockaddr* nodeAddress = new sockaddr;
ssize_t receivedBytes = 0;
unsigned char* packetData = new unsigned char[MAX_PACKET_SIZE];
uint16_t nodeID = 0;
Node* avatarNode = NULL;
timeval lastDomainServerCheckIn = {};
// we only need to hear back about avatar nodes from the DS
NodeList::getInstance()->setNodeTypesOfInterest(&NODE_TYPE_AGENT, 1);
while (true) {
// send a check in packet to the domain server if DOMAIN_SERVER_CHECK_IN_USECS has elapsed
if (usecTimestampNow() - usecTimestamp(&lastDomainServerCheckIn) >= DOMAIN_SERVER_CHECK_IN_USECS) {
gettimeofday(&lastDomainServerCheckIn, NULL);
NodeList::getInstance()->sendDomainServerCheckIn();
}
if (nodeList->getNodeSocket()->receive(nodeAddress, packetData, &receivedBytes) &&
packetVersionMatch(packetData)) {
switch (packetData[0]) {
case PACKET_TYPE_HEAD_DATA:
// grab the node ID from the packet
unpackNodeId(packetData + numBytesForPacketHeader(packetData), &nodeID);
// add or update the node in our list
avatarNode = nodeList->addOrUpdateNode(nodeAddress, nodeAddress, NODE_TYPE_AGENT, nodeID);
// parse positional data from an node
nodeList->updateNodeWithData(avatarNode, packetData, receivedBytes);
case PACKET_TYPE_INJECT_AUDIO:
broadcastAvatarData(nodeList, nodeAddress);
break;
case PACKET_TYPE_AVATAR_VOXEL_URL:
case PACKET_TYPE_AVATAR_FACE_VIDEO:
// grab the node ID from the packet
unpackNodeId(packetData + numBytesForPacketHeader(packetData), &nodeID);
// let everyone else know about the update
for (NodeList::iterator node = nodeList->begin(); node != nodeList->end(); node++) {
if (node->getActiveSocket() && node->getNodeID() != nodeID) {
nodeList->getNodeSocket()->send(node->getActiveSocket(), packetData, receivedBytes);
}
}
break;
case PACKET_TYPE_DOMAIN:
// ignore the DS packet, for now nodes are added only when they communicate directly with us
break;
default:
// hand this off to the NodeList
nodeList->processNodeData(nodeAddress, packetData, receivedBytes);
break;
}
}
}
nodeList->stopSilentNodeRemovalThread();
return 0;
}
int main(int argc, const char* argv[]) {
qInstallMessageHandler(Logging::verboseMessageHandler);
NodeList* nodeList = NodeList::createInstance(NODE_TYPE_DOMAIN, DOMAIN_LISTEN_PORT);
setvbuf(stdout, NULL, _IOLBF, 0);
ssize_t receivedBytes = 0;
char nodeType = '\0';
unsigned char broadcastPacket[MAX_PACKET_SIZE];
unsigned char* currentBufferPos;
unsigned char* startPointer;
sockaddr_in nodePublicAddress, nodeLocalAddress, replyDestinationSocket;
nodeLocalAddress.sin_family = AF_INET;
in_addr_t serverLocalAddress = getLocalAddress();
nodeList->startSilentNodeRemovalThread();
timeval lastStatSendTime = {};
const char ASSIGNMENT_SERVER_OPTION[] = "-a";
// grab the overriden assignment-server hostname from argv, if it exists
const char* customAssignmentServer = getCmdOption(argc, argv, ASSIGNMENT_SERVER_OPTION);
if (customAssignmentServer) {
sockaddr_in customAssignmentSocket = socketForHostnameAndHostOrderPort(customAssignmentServer, ASSIGNMENT_SERVER_PORT);
nodeList->setAssignmentServerSocket((sockaddr*) &customAssignmentSocket);
}
// use a map to keep track of iterations of silence for assignment creation requests
const long long GLOBAL_ASSIGNMENT_REQUEST_INTERVAL_USECS = 1 * 1000 * 1000;
timeval lastGlobalAssignmentRequest = {};
// as a domain-server we will always want an audio mixer and avatar mixer
// setup the create assignments for those
Assignment audioMixerAssignment(Assignment::CreateCommand,
Assignment::AudioMixerType,
Assignment::LocalLocation);
Assignment avatarMixerAssignment(Assignment::CreateCommand,
Assignment::AvatarMixerType,
Assignment::LocalLocation);
// construct a local socket to send with our created assignments to the global AS
sockaddr_in localSocket = {};
localSocket.sin_family = AF_INET;
localSocket.sin_port = htons(nodeList->getInstance()->getNodeSocket()->getListeningPort());
localSocket.sin_addr.s_addr = serverLocalAddress;
// setup the mongoose web server
struct mg_context *ctx;
struct mg_callbacks callbacks = {};
// list of options. Last element must be NULL.
const char *options[] = {"listening_ports", "8080",
"document_root", "./resources/web", NULL};
callbacks.begin_request = mongooseRequestHandler;
callbacks.upload = mongooseUploadHandler;
// Start the web server.
ctx = mg_start(&callbacks, NULL, options);
while (true) {
::assignmentQueueMutex.lock();
// check if our audio-mixer or avatar-mixer are dead and we don't have existing assignments in the queue
// so we can add those assignments back to the front of the queue since they are high-priority
if (!nodeList->soloNodeOfType(NODE_TYPE_AVATAR_MIXER) &&
std::find(::assignmentQueue.begin(), assignmentQueue.end(), &avatarMixerAssignment) == ::assignmentQueue.end()) {
qDebug("Missing an avatar mixer and assignment not in queue. Adding.\n");
::assignmentQueue.push_front(&avatarMixerAssignment);
}
if (!nodeList->soloNodeOfType(NODE_TYPE_AUDIO_MIXER) &&
std::find(::assignmentQueue.begin(), ::assignmentQueue.end(), &audioMixerAssignment) == ::assignmentQueue.end()) {
qDebug("Missing an audio mixer and assignment not in queue. Adding.\n");
::assignmentQueue.push_front(&audioMixerAssignment);
}
::assignmentQueueMutex.unlock();
while (nodeList->getNodeSocket()->receive((sockaddr *)&nodePublicAddress, packetData, &receivedBytes) &&
packetVersionMatch(packetData)) {
if (packetData[0] == PACKET_TYPE_DOMAIN_REPORT_FOR_DUTY || packetData[0] == PACKET_TYPE_DOMAIN_LIST_REQUEST) {
// this is an RFD or domain list request packet, and there is a version match
std::map<char, Node *> newestSoloNodes;
int numBytesSenderHeader = numBytesForPacketHeader(packetData);
nodeType = *(packetData + numBytesSenderHeader);
int numBytesSocket = unpackSocket(packetData + numBytesSenderHeader + sizeof(NODE_TYPE),
(sockaddr*) &nodeLocalAddress);
replyDestinationSocket = nodePublicAddress;
// check the node public address
// if it matches our local address
// or if it's the loopback address we're on the same box
if (nodePublicAddress.sin_addr.s_addr == serverLocalAddress ||
nodePublicAddress.sin_addr.s_addr == htonl(INADDR_LOOPBACK)) {
nodePublicAddress.sin_addr.s_addr = 0;
}
Node* newNode = nodeList->addOrUpdateNode((sockaddr*) &nodePublicAddress,
(sockaddr*) &nodeLocalAddress,
nodeType,
nodeList->getLastNodeID());
// if addOrUpdateNode returns NULL this was a solo node we already have, don't talk back to it
if (newNode) {
if (newNode->getNodeID() == nodeList->getLastNodeID()) {
nodeList->increaseNodeID();
}
int numHeaderBytes = populateTypeAndVersion(broadcastPacket, PACKET_TYPE_DOMAIN);
currentBufferPos = broadcastPacket + numHeaderBytes;
startPointer = currentBufferPos;
unsigned char* nodeTypesOfInterest = packetData + numBytesSenderHeader + sizeof(NODE_TYPE)
+ numBytesSocket + sizeof(unsigned char);
int numInterestTypes = *(nodeTypesOfInterest - 1);
if (numInterestTypes > 0) {
// if the node has sent no types of interest, assume they want nothing but their own ID back
for (NodeList::iterator node = nodeList->begin(); node != nodeList->end(); node++) {
if (!node->matches((sockaddr*) &nodePublicAddress, (sockaddr*) &nodeLocalAddress, nodeType) &&
memchr(nodeTypesOfInterest, node->getType(), numInterestTypes)) {
// this is not the node themselves
// and this is an node of a type in the passed node types of interest
// or the node did not pass us any specific types they are interested in
if (memchr(SOLO_NODE_TYPES, node->getType(), sizeof(SOLO_NODE_TYPES)) == NULL) {
// this is an node of which there can be multiple, just add them to the packet
// don't send avatar nodes to other avatars, that will come from avatar mixer
if (nodeType != NODE_TYPE_AGENT || node->getType() != NODE_TYPE_AGENT) {
currentBufferPos = addNodeToBroadcastPacket(currentBufferPos, &(*node));
}
} else {
// solo node, we need to only send newest
if (newestSoloNodes[node->getType()] == NULL ||
newestSoloNodes[node->getType()]->getWakeMicrostamp() < node->getWakeMicrostamp()) {
// we have to set the newer solo node to add it to the broadcast later
newestSoloNodes[node->getType()] = &(*node);
}
}
}
}
for (std::map<char, Node *>::iterator soloNode = newestSoloNodes.begin();
soloNode != newestSoloNodes.end();
soloNode++) {
// this is the newest alive solo node, add them to the packet
currentBufferPos = addNodeToBroadcastPacket(currentBufferPos, soloNode->second);
}
}
// update last receive to now
uint64_t timeNow = usecTimestampNow();
newNode->setLastHeardMicrostamp(timeNow);
if (packetData[0] == PACKET_TYPE_DOMAIN_REPORT_FOR_DUTY
&& memchr(SOLO_NODE_TYPES, nodeType, sizeof(SOLO_NODE_TYPES))) {
newNode->setWakeMicrostamp(timeNow);
}
// add the node ID to the end of the pointer
currentBufferPos += packNodeId(currentBufferPos, newNode->getNodeID());
// send the constructed list back to this node
nodeList->getNodeSocket()->send((sockaddr*)&replyDestinationSocket,
broadcastPacket,
(currentBufferPos - startPointer) + numHeaderBytes);
}
} else if (packetData[0] == PACKET_TYPE_REQUEST_ASSIGNMENT) {
qDebug("Received a request for assignment.\n");
::assignmentQueueMutex.lock();
// this is an unassigned client talking to us directly for an assignment
// go through our queue and see if there are any assignments to give out
std::deque<Assignment*>::iterator assignment = ::assignmentQueue.begin();
while (assignment != ::assignmentQueue.end()) {
// give this assignment out, no conditions stop us from giving it to the local assignment client
int numHeaderBytes = populateTypeAndVersion(broadcastPacket, PACKET_TYPE_CREATE_ASSIGNMENT);
int numAssignmentBytes = (*assignment)->packToBuffer(broadcastPacket + numHeaderBytes);
nodeList->getNodeSocket()->send((sockaddr*) &nodePublicAddress,
broadcastPacket,
numHeaderBytes + numAssignmentBytes);
// remove the assignment from the queue
::assignmentQueue.erase(assignment);
if ((*assignment)->getType() == Assignment::AgentType) {
// if this is a script assignment we need to delete it to avoid a memory leak
delete *assignment;
}
// stop looping, we've handed out an assignment
break;
}
::assignmentQueueMutex.unlock();
}
}
// if ASSIGNMENT_REQUEST_INTERVAL_USECS have passed since last global assignment request then fire off another
if (usecTimestampNow() - usecTimestamp(&lastGlobalAssignmentRequest) >= GLOBAL_ASSIGNMENT_REQUEST_INTERVAL_USECS) {
gettimeofday(&lastGlobalAssignmentRequest, NULL);
::assignmentQueueMutex.lock();
// go through our queue and see if there are any assignments to send to the global assignment server
std::deque<Assignment*>::iterator assignment = ::assignmentQueue.begin();
while (assignment != assignmentQueue.end()) {
if ((*assignment)->getLocation() != Assignment::LocalLocation) {
// attach our local socket to the assignment so the assignment-server can optionally hand it out
(*assignment)->setAttachedLocalSocket((sockaddr*) &localSocket);
nodeList->sendAssignment(*(*assignment));
// remove the assignment from the queue
::assignmentQueue.erase(assignment);
if ((*assignment)->getType() == Assignment::AgentType) {
// if this is a script assignment we need to delete it to avoid a memory leak
delete *assignment;
}
// stop looping, we've handed out an assignment
break;
} else {
// push forward the iterator to check the next assignment
assignment++;
}
}
::assignmentQueueMutex.unlock();
}
if (Logging::shouldSendStats()) {
if (usecTimestampNow() - usecTimestamp(&lastStatSendTime) >= (NODE_COUNT_STAT_INTERVAL_MSECS * 1000)) {
// time to send our count of nodes and servers to logstash
const char NODE_COUNT_LOGSTASH_KEY[] = "ds-node-count";
Logging::stashValue(STAT_TYPE_TIMER, NODE_COUNT_LOGSTASH_KEY, nodeList->getNumAliveNodes());
gettimeofday(&lastStatSendTime, NULL);
}
}
}
return 0;
}
int main(int argc, const char * argv[]) {
pthread_mutex_init(&::treeLock, NULL);
qInstallMessageHandler(sharedMessageHandler);
NodeList* nodeList = NodeList::createInstance(NODE_TYPE_VOXEL_SERVER, VOXEL_LISTEN_PORT);
setvbuf(stdout, NULL, _IOLBF, 0);
// Handle Local Domain testing with the --local command line
const char* local = "--local";
::wantLocalDomain = cmdOptionExists(argc, argv,local);
if (::wantLocalDomain) {
printf("Local Domain MODE!\n");
nodeList->setDomainIPToLocalhost();
}
nodeList->linkedDataCreateCallback = &attachVoxelNodeDataToNode;
nodeList->startSilentNodeRemovalThread();
srand((unsigned)time(0));
const char* DISPLAY_VOXEL_STATS = "--displayVoxelStats";
::displayVoxelStats = cmdOptionExists(argc, argv, DISPLAY_VOXEL_STATS);
printf("displayVoxelStats=%s\n", debug::valueOf(::displayVoxelStats));
const char* DEBUG_VOXEL_SENDING = "--debugVoxelSending";
::debugVoxelSending = cmdOptionExists(argc, argv, DEBUG_VOXEL_SENDING);
printf("debugVoxelSending=%s\n", debug::valueOf(::debugVoxelSending));
const char* DEBUG_VOXEL_RECEIVING = "--debugVoxelReceiving";
::debugVoxelReceiving = cmdOptionExists(argc, argv, DEBUG_VOXEL_RECEIVING);
printf("debugVoxelReceiving=%s\n", debug::valueOf(::debugVoxelReceiving));
const char* WANT_ANIMATION_DEBUG = "--shouldShowAnimationDebug";
::shouldShowAnimationDebug = cmdOptionExists(argc, argv, WANT_ANIMATION_DEBUG);
printf("shouldShowAnimationDebug=%s\n", debug::valueOf(::shouldShowAnimationDebug));
const char* WANT_COLOR_RANDOMIZER = "--wantColorRandomizer";
::wantColorRandomizer = cmdOptionExists(argc, argv, WANT_COLOR_RANDOMIZER);
printf("wantColorRandomizer=%s\n", debug::valueOf(::wantColorRandomizer));
// By default we will voxel persist, if you want to disable this, then pass in this parameter
const char* NO_VOXEL_PERSIST = "--NoVoxelPersist";
if (cmdOptionExists(argc, argv, NO_VOXEL_PERSIST)) {
::wantVoxelPersist = false;
}
printf("wantVoxelPersist=%s\n", debug::valueOf(::wantVoxelPersist));
// if we want Voxel Persistance, load the local file now...
bool persistantFileRead = false;
if (::wantVoxelPersist) {
printf("loading voxels from file...\n");
persistantFileRead = ::serverTree.readFromSVOFile(::wantLocalDomain ? LOCAL_VOXELS_PERSIST_FILE : VOXELS_PERSIST_FILE);
if (persistantFileRead) {
PerformanceWarning warn(::shouldShowAnimationDebug,
"persistVoxelsWhenDirty() - reaverageVoxelColors()", ::shouldShowAnimationDebug);
// after done inserting all these voxels, then reaverage colors
serverTree.reaverageVoxelColors(serverTree.rootNode);
printf("Voxels reAveraged\n");
}
::serverTree.clearDirtyBit(); // the tree is clean since we just loaded it
printf("DONE loading voxels from file... fileRead=%s\n", debug::valueOf(persistantFileRead));
unsigned long nodeCount = ::serverTree.rootNode->getSubTreeNodeCount();
unsigned long internalNodeCount = ::serverTree.rootNode->getSubTreeInternalNodeCount();
unsigned long leafNodeCount = ::serverTree.rootNode->getSubTreeLeafNodeCount();
printf("Nodes after loading scene %lu nodes %lu internal %lu leaves\n", nodeCount, internalNodeCount, leafNodeCount);
}
// Check to see if the user passed in a command line option for loading an old style local
// Voxel File. If so, load it now. This is not the same as a voxel persist file
const char* INPUT_FILE = "-i";
const char* voxelsFilename = getCmdOption(argc, argv, INPUT_FILE);
if (voxelsFilename) {
serverTree.readFromSVOFile(voxelsFilename);
}
// Check to see if the user passed in a command line option for setting packet send rate
const char* PACKETS_PER_SECOND = "--packetsPerSecond";
const char* packetsPerSecond = getCmdOption(argc, argv, PACKETS_PER_SECOND);
if (packetsPerSecond) {
PACKETS_PER_CLIENT_PER_INTERVAL = atoi(packetsPerSecond)/10;
if (PACKETS_PER_CLIENT_PER_INTERVAL < 1) {
PACKETS_PER_CLIENT_PER_INTERVAL = 1;
}
printf("packetsPerSecond=%s PACKETS_PER_CLIENT_PER_INTERVAL=%d\n", packetsPerSecond, PACKETS_PER_CLIENT_PER_INTERVAL);
}
const char* ADD_RANDOM_VOXELS = "--AddRandomVoxels";
if (cmdOptionExists(argc, argv, ADD_RANDOM_VOXELS)) {
// create an octal code buffer and load it with 0 so that the recursive tree fill can give
// octal codes to the tree nodes that it is creating
randomlyFillVoxelTree(MAX_VOXEL_TREE_DEPTH_LEVELS, serverTree.rootNode);
}
const char* ADD_SCENE = "--AddScene";
bool addScene = cmdOptionExists(argc, argv, ADD_SCENE);
const char* NO_ADD_SCENE = "--NoAddScene";
bool noAddScene = cmdOptionExists(argc, argv, NO_ADD_SCENE);
if (addScene && noAddScene) {
printf("WARNING! --AddScene and --NoAddScene are mutually exclusive. We will honor --NoAddScene\n");
}
// We will add a scene if...
// 1) we attempted to load a persistant file and it wasn't there
// 2) you asked us to add a scene
// HOWEVER -- we will NEVER add a scene if you explicitly tell us not to!
//
// TEMPORARILY DISABLED!!!
bool actuallyAddScene = false; // !noAddScene && (addScene || (::wantVoxelPersist && !persistantFileRead));
if (actuallyAddScene) {
addSphereScene(&serverTree);
}
// for now, initialize the environments with fixed values
environmentData[1].setID(1);
environmentData[1].setGravity(1.0f);
environmentData[1].setAtmosphereCenter(glm::vec3(0.5, 0.5, (0.25 - 0.06125)) * (float)TREE_SCALE);
environmentData[1].setAtmosphereInnerRadius(0.030625f * TREE_SCALE);
environmentData[1].setAtmosphereOuterRadius(0.030625f * TREE_SCALE * 1.05f);
environmentData[2].setID(2);
environmentData[2].setGravity(1.0f);
environmentData[2].setAtmosphereCenter(glm::vec3(0.5f, 0.5f, 0.5f) * (float)TREE_SCALE);
environmentData[2].setAtmosphereInnerRadius(0.1875f * TREE_SCALE);
environmentData[2].setAtmosphereOuterRadius(0.1875f * TREE_SCALE * 1.05f);
environmentData[2].setScatteringWavelengths(glm::vec3(0.475f, 0.570f, 0.650f)); // swaps red and blue
pthread_t sendVoxelThread;
pthread_create(&sendVoxelThread, NULL, distributeVoxelsToListeners, NULL);
sockaddr nodePublicAddress;
unsigned char *packetData = new unsigned char[MAX_PACKET_SIZE];
ssize_t receivedBytes;
timeval lastDomainServerCheckIn = {};
// loop to send to nodes requesting data
while (true) {
// send a check in packet to the domain server if DOMAIN_SERVER_CHECK_IN_USECS has elapsed
if (usecTimestampNow() - usecTimestamp(&lastDomainServerCheckIn) >= DOMAIN_SERVER_CHECK_IN_USECS) {
gettimeofday(&lastDomainServerCheckIn, NULL);
NodeList::getInstance()->sendDomainServerCheckIn();
}
// check to see if we need to persist our voxel state
persistVoxelsWhenDirty();
if (nodeList->getNodeSocket()->receive(&nodePublicAddress, packetData, &receivedBytes) &&
packetVersionMatch(packetData)) {
int numBytesPacketHeader = numBytesForPacketHeader(packetData);
if (packetData[0] == PACKET_TYPE_SET_VOXEL || packetData[0] == PACKET_TYPE_SET_VOXEL_DESTRUCTIVE) {
bool destructive = (packetData[0] == PACKET_TYPE_SET_VOXEL_DESTRUCTIVE);
PerformanceWarning warn(::shouldShowAnimationDebug,
destructive ? "PACKET_TYPE_SET_VOXEL_DESTRUCTIVE" : "PACKET_TYPE_SET_VOXEL",
::shouldShowAnimationDebug);
::receivedPacketCount++;
unsigned short int itemNumber = (*((unsigned short int*)(packetData + numBytesPacketHeader)));
if (::shouldShowAnimationDebug) {
printf("got %s - command from client receivedBytes=%ld itemNumber=%d\n",
destructive ? "PACKET_TYPE_SET_VOXEL_DESTRUCTIVE" : "PACKET_TYPE_SET_VOXEL",
receivedBytes,itemNumber);
}
if (::debugVoxelReceiving) {
printf("got %s - %d command from client receivedBytes=%ld itemNumber=%d\n",
destructive ? "PACKET_TYPE_SET_VOXEL_DESTRUCTIVE" : "PACKET_TYPE_SET_VOXEL",
::receivedPacketCount, receivedBytes,itemNumber);
}
int atByte = numBytesPacketHeader + sizeof(itemNumber);
unsigned char* voxelData = (unsigned char*)&packetData[atByte];
while (atByte < receivedBytes) {
unsigned char octets = (unsigned char)*voxelData;
const int COLOR_SIZE_IN_BYTES = 3;
int voxelDataSize = bytesRequiredForCodeLength(octets) + COLOR_SIZE_IN_BYTES;
int voxelCodeSize = bytesRequiredForCodeLength(octets);
// color randomization on insert
int colorRandomizer = ::wantColorRandomizer ? randIntInRange (-50, 50) : 0;
int red = voxelData[voxelCodeSize + 0];
int green = voxelData[voxelCodeSize + 1];
int blue = voxelData[voxelCodeSize + 2];
if (::shouldShowAnimationDebug) {
printf("insert voxels - wantColorRandomizer=%s old r=%d,g=%d,b=%d \n",
(::wantColorRandomizer?"yes":"no"),red,green,blue);
}
red = std::max(0, std::min(255, red + colorRandomizer));
green = std::max(0, std::min(255, green + colorRandomizer));
blue = std::max(0, std::min(255, blue + colorRandomizer));
if (::shouldShowAnimationDebug) {
printf("insert voxels - wantColorRandomizer=%s NEW r=%d,g=%d,b=%d \n",
(::wantColorRandomizer?"yes":"no"),red,green,blue);
}
voxelData[voxelCodeSize + 0] = red;
voxelData[voxelCodeSize + 1] = green;
voxelData[voxelCodeSize + 2] = blue;
if (::shouldShowAnimationDebug) {
float* vertices = firstVertexForCode(voxelData);
printf("inserting voxel at: %f,%f,%f\n", vertices[0], vertices[1], vertices[2]);
delete []vertices;
}
serverTree.readCodeColorBufferToTree(voxelData, destructive);
// skip to next
voxelData += voxelDataSize;
atByte += voxelDataSize;
}
} else if (packetData[0] == PACKET_TYPE_ERASE_VOXEL) {
// Send these bits off to the VoxelTree class to process them
pthread_mutex_lock(&::treeLock);
serverTree.processRemoveVoxelBitstream((unsigned char*)packetData, receivedBytes);
pthread_mutex_unlock(&::treeLock);
} else if (packetData[0] == PACKET_TYPE_Z_COMMAND) {
// the Z command is a special command that allows the sender to send the voxel server high level semantic
// requests, like erase all, or add sphere scene
char* command = (char*) &packetData[numBytesPacketHeader]; // start of the command
int commandLength = strlen(command); // commands are null terminated strings
int totalLength = numBytesPacketHeader + commandLength + 1; // 1 for null termination
printf("got Z message len(%ld)= %s\n", receivedBytes, command);
bool rebroadcast = true; // by default rebroadcast
while (totalLength <= receivedBytes) {
if (strcmp(command, ERASE_ALL_COMMAND) == 0) {
printf("got Z message == erase all\n");
eraseVoxelTreeAndCleanupNodeVisitData();
rebroadcast = false;
}
if (strcmp(command, ADD_SCENE_COMMAND) == 0) {
printf("got Z message == add scene\n");
addSphereScene(&serverTree);
rebroadcast = false;
}
if (strcmp(command, TEST_COMMAND) == 0) {
printf("got Z message == a message, nothing to do, just report\n");
}
totalLength += commandLength + 1; // 1 for null termination
}
if (rebroadcast) {
// Now send this to the connected nodes so they can also process these messages
printf("rebroadcasting Z message to connected nodes... nodeList.broadcastToNodes()\n");
nodeList->broadcastToNodes(packetData, receivedBytes, &NODE_TYPE_AGENT, 1);
}
} else if (packetData[0] == PACKET_TYPE_HEAD_DATA) {
// If we got a PACKET_TYPE_HEAD_DATA, then we're talking to an NODE_TYPE_AVATAR, and we
// need to make sure we have it in our nodeList.
uint16_t nodeID = 0;
unpackNodeId(packetData + numBytesPacketHeader, &nodeID);
Node* node = nodeList->addOrUpdateNode(&nodePublicAddress,
&nodePublicAddress,
NODE_TYPE_AGENT,
nodeID);
nodeList->updateNodeWithData(node, packetData, receivedBytes);
} else if (packetData[0] == PACKET_TYPE_PING) {
// If the packet is a ping, let processNodeData handle it.
nodeList->processNodeData(&nodePublicAddress, packetData, receivedBytes);
}
}
}
pthread_join(sendVoxelThread, NULL);
pthread_mutex_destroy(&::treeLock);
return 0;
}
void AvatarMixer::run() {
// change the logging target name while AvatarMixer is running
Logging::setTargetName(AVATAR_MIXER_LOGGING_NAME);
NodeList* nodeList = NodeList::getInstance();
nodeList->setOwnerType(NODE_TYPE_AVATAR_MIXER);
nodeList->linkedDataCreateCallback = attachAvatarDataToNode;
nodeList->startSilentNodeRemovalThread();
sockaddr* nodeAddress = new sockaddr;
ssize_t receivedBytes = 0;
unsigned char* packetData = new unsigned char[MAX_PACKET_SIZE];
uint16_t nodeID = 0;
Node* avatarNode = NULL;
timeval lastDomainServerCheckIn = {};
// we only need to hear back about avatar nodes from the DS
nodeList->setNodeTypesOfInterest(&NODE_TYPE_AGENT, 1);
while (true) {
if (NodeList::getInstance()->getNumNoReplyDomainCheckIns() == MAX_SILENT_DOMAIN_SERVER_CHECK_INS) {
break;
}
// send a check in packet to the domain server if DOMAIN_SERVER_CHECK_IN_USECS has elapsed
if (usecTimestampNow() - usecTimestamp(&lastDomainServerCheckIn) >= DOMAIN_SERVER_CHECK_IN_USECS) {
gettimeofday(&lastDomainServerCheckIn, NULL);
NodeList::getInstance()->sendDomainServerCheckIn();
}
if (nodeList->getNodeSocket()->receive(nodeAddress, packetData, &receivedBytes) &&
packetVersionMatch(packetData)) {
switch (packetData[0]) {
case PACKET_TYPE_HEAD_DATA:
// grab the node ID from the packet
unpackNodeId(packetData + numBytesForPacketHeader(packetData), &nodeID);
// add or update the node in our list
avatarNode = nodeList->addOrUpdateNode(nodeAddress, nodeAddress, NODE_TYPE_AGENT, nodeID);
// parse positional data from an node
nodeList->updateNodeWithData(avatarNode, packetData, receivedBytes);
case PACKET_TYPE_INJECT_AUDIO:
broadcastAvatarData(nodeList, nodeAddress);
break;
case PACKET_TYPE_AVATAR_VOXEL_URL:
case PACKET_TYPE_AVATAR_FACE_VIDEO:
// grab the node ID from the packet
unpackNodeId(packetData + numBytesForPacketHeader(packetData), &nodeID);
// let everyone else know about the update
for (NodeList::iterator node = nodeList->begin(); node != nodeList->end(); node++) {
if (node->getActiveSocket() && node->getNodeID() != nodeID) {
nodeList->getNodeSocket()->send(node->getActiveSocket(), packetData, receivedBytes);
}
}
break;
default:
// hand this off to the NodeList
nodeList->processNodeData(nodeAddress, packetData, receivedBytes);
break;
}
}
}
nodeList->stopSilentNodeRemovalThread();
}
int main(int argc, const char * argv[])
{
NodeList* nodeList = NodeList::createInstance(NODE_TYPE_DOMAIN, DOMAIN_LISTEN_PORT);
// If user asks to run in "local" mode then we do NOT replace the IP
// with the EC2 IP. Otherwise, we will replace the IP like we used to
// this allows developers to run a local domain without recompiling the
// domain server
bool isLocalMode = cmdOptionExists(argc, argv, "--local");
if (isLocalMode) {
printf("NOTE: Running in Local Mode!\n");
} else {
printf("--------------------------------------------------\n");
printf("NOTE: Running in EC2 Mode. \n");
printf("If you're a developer testing a local system, you\n");
printf("probably want to include --local on command line.\n");
printf("--------------------------------------------------\n");
}
setvbuf(stdout, NULL, _IOLBF, 0);
ssize_t receivedBytes = 0;
char nodeType = '\0';
unsigned char broadcastPacket[MAX_PACKET_SIZE];
int numHeaderBytes = populateTypeAndVersion(broadcastPacket, PACKET_TYPE_DOMAIN);
unsigned char* currentBufferPos;
unsigned char* startPointer;
sockaddr_in nodePublicAddress, nodeLocalAddress;
nodeLocalAddress.sin_family = AF_INET;
in_addr_t serverLocalAddress = getLocalAddress();
nodeList->startSilentNodeRemovalThread();
timeval lastStatSendTime = {};
while (true) {
if (nodeList->getNodeSocket()->receive((sockaddr *)&nodePublicAddress, packetData, &receivedBytes) &&
(packetData[0] == PACKET_TYPE_DOMAIN_REPORT_FOR_DUTY || packetData[0] == PACKET_TYPE_DOMAIN_LIST_REQUEST) &&
packetVersionMatch(packetData)) {
// this is an RFD or domain list request packet, and there is a version match
std::map<char, Node *> newestSoloNodes;
int numBytesSenderHeader = numBytesForPacketHeader(packetData);
nodeType = *(packetData + numBytesSenderHeader);
int numBytesSocket = unpackSocket(packetData + numBytesSenderHeader + sizeof(NODE_TYPE),
(sockaddr*) &nodeLocalAddress);
sockaddr* destinationSocket = (sockaddr*) &nodePublicAddress;
// check the node public address
// if it matches our local address we're on the same box
// so hardcode the EC2 public address for now
if (nodePublicAddress.sin_addr.s_addr == serverLocalAddress) {
// If we're not running "local" then we do replace the IP
// with the EC2 IP. Otherwise, we use our normal public IP
if (!isLocalMode) {
nodePublicAddress.sin_addr.s_addr = 895283510; // local IP in this format...
destinationSocket = (sockaddr*) &nodeLocalAddress;
}
}
Node* newNode = nodeList->addOrUpdateNode((sockaddr*) &nodePublicAddress,
(sockaddr*) &nodeLocalAddress,
nodeType,
nodeList->getLastNodeID());
if (newNode->getNodeID() == nodeList->getLastNodeID()) {
nodeList->increaseNodeID();
}
currentBufferPos = broadcastPacket + numHeaderBytes;
startPointer = currentBufferPos;
unsigned char* nodeTypesOfInterest = packetData + numBytesSenderHeader + sizeof(NODE_TYPE)
+ numBytesSocket + sizeof(unsigned char);
int numInterestTypes = *(nodeTypesOfInterest - 1);
if (numInterestTypes > 0) {
// if the node has sent no types of interest, assume they want nothing but their own ID back
for (NodeList::iterator node = nodeList->begin(); node != nodeList->end(); node++) {
if (!node->matches((sockaddr*) &nodePublicAddress, (sockaddr*) &nodeLocalAddress, nodeType) &&
memchr(nodeTypesOfInterest, node->getType(), numInterestTypes)) {
// this is not the node themselves
// and this is an node of a type in the passed node types of interest
// or the node did not pass us any specific types they are interested in
if (memchr(SOLO_NODE_TYPES, node->getType(), sizeof(SOLO_NODE_TYPES)) == NULL) {
// this is an node of which there can be multiple, just add them to the packet
// don't send avatar nodes to other avatars, that will come from avatar mixer
if (nodeType != NODE_TYPE_AGENT || node->getType() != NODE_TYPE_AGENT) {
currentBufferPos = addNodeToBroadcastPacket(currentBufferPos, &(*node));
}
} else {
// solo node, we need to only send newest
if (newestSoloNodes[node->getType()] == NULL ||
newestSoloNodes[node->getType()]->getWakeMicrostamp() < node->getWakeMicrostamp()) {
// we have to set the newer solo node to add it to the broadcast later
newestSoloNodes[node->getType()] = &(*node);
}
}
}
}
for (std::map<char, Node *>::iterator soloNode = newestSoloNodes.begin();
soloNode != newestSoloNodes.end();
soloNode++) {
// this is the newest alive solo node, add them to the packet
currentBufferPos = addNodeToBroadcastPacket(currentBufferPos, soloNode->second);
}
}
// update last receive to now
uint64_t timeNow = usecTimestampNow();
newNode->setLastHeardMicrostamp(timeNow);
if (packetData[0] == PACKET_TYPE_DOMAIN_REPORT_FOR_DUTY
&& memchr(SOLO_NODE_TYPES, nodeType, sizeof(SOLO_NODE_TYPES))) {
newNode->setWakeMicrostamp(timeNow);
}
// add the node ID to the end of the pointer
currentBufferPos += packNodeId(currentBufferPos, newNode->getNodeID());
// send the constructed list back to this node
nodeList->getNodeSocket()->send(destinationSocket,
broadcastPacket,
(currentBufferPos - startPointer) + numHeaderBytes);
}
if (Logstash::shouldSendStats()) {
if (usecTimestampNow() - usecTimestamp(&lastStatSendTime) >= (NODE_COUNT_STAT_INTERVAL_MSECS * 1000)) {
// time to send our count of nodes and servers to logstash
const char NODE_COUNT_LOGSTASH_KEY[] = "ds-node-count";
Logstash::stashValue(STAT_TYPE_TIMER, NODE_COUNT_LOGSTASH_KEY, nodeList->getNumAliveNodes());
gettimeofday(&lastStatSendTime, NULL);
}
}
}
return 0;
}