void DomainServer::possiblyAddStaticAssignmentsBackToQueueAfterRestart(timeval* startTime) {
// if the domain-server has just restarted,
// check if there are static assignments in the file that we need to
// throw into the assignment queue
const uint64_t RESTART_HOLD_TIME_USECS = 5 * 1000 * 1000;
if (!_hasCompletedRestartHold && usecTimestampNow() - usecTimestamp(startTime) > RESTART_HOLD_TIME_USECS) {
_hasCompletedRestartHold = true;
// pull anything in the static assignment file that isn't spoken for and add to the assignment queue
for (int i = 0; i < MAX_STATIC_ASSIGNMENT_FILE_ASSIGNMENTS; i++) {
if (_staticAssignments[i].getUUID().isNull()) {
// reached the end of static assignments, bail
break;
}
bool foundMatchingAssignment = false;
NodeList* nodeList = NodeList::getInstance();
// enumerate the nodes and check if there is one with an attached assignment with matching UUID
for (NodeList::iterator node = nodeList->begin(); node != nodeList->end(); node++) {
if (node->getLinkedData()) {
Assignment* linkedAssignment = (Assignment*) node->getLinkedData();
if (linkedAssignment->getUUID() == _staticAssignments[i].getUUID()) {
foundMatchingAssignment = true;
break;
}
}
}
if (!foundMatchingAssignment) {
// this assignment has not been fulfilled - reset the UUID and add it to the assignment queue
_staticAssignments[i].resetUUID();
qDebug() << "Adding static assignment to queue -" << _staticAssignments[i] << "\n";
_assignmentQueueMutex.lock();
_assignmentQueue.push_back(&_staticAssignments[i]);
_assignmentQueueMutex.unlock();
}
}
}
}
bool DomainServer::checkInWithUUIDMatchesExistingNode(sockaddr* nodePublicSocket,
sockaddr* nodeLocalSocket,
const uchar* checkInData) {
// pull the UUID passed with the check in
QUuid checkInUUID = QUuid::fromRfc4122(QByteArray((const char*) checkInData + numBytesForPacketHeader(checkInData) +
sizeof(NODE_TYPE),
NUM_BYTES_RFC4122_UUID));
NodeList* nodeList = NodeList::getInstance();
for (NodeList::iterator node = nodeList->begin(); node != nodeList->end(); node++) {
if (node->getLinkedData()
&& socketMatch(node->getPublicSocket(), nodePublicSocket)
&& socketMatch(node->getLocalSocket(), nodeLocalSocket)
&& ((Assignment*) node->getLinkedData())->getUUID() == checkInUUID) {
// this is a matching existing node if the public socket, local socket, and UUID match
return true;
}
}
return false;
}
void eraseVoxelTreeAndCleanupNodeVisitData() {
// As our tree to erase all it's voxels
::serverTree.eraseAllVoxels();
// enumerate the nodes clean up their marker nodes
for (NodeList::iterator node = NodeList::getInstance()->begin(); node != NodeList::getInstance()->end(); node++) {
VoxelNodeData* nodeData = (VoxelNodeData*) node->getLinkedData();
if (nodeData) {
// clean up the node visit data
nodeData->nodeBag.deleteAll();
}
}
}
bool DomainServer::checkInWithUUIDMatchesExistingNode(sockaddr* nodePublicSocket,
sockaddr* nodeLocalSocket,
const QUuid& checkInUUID) {
NodeList* nodeList = NodeList::getInstance();
for (NodeList::iterator node = nodeList->begin(); node != nodeList->end(); node++) {
if (node->getLinkedData()
&& socketMatch(node->getPublicSocket(), nodePublicSocket)
&& socketMatch(node->getLocalSocket(), nodeLocalSocket)
&& node->getUUID() == checkInUUID) {
// this is a matching existing node if the public socket, local socket, and UUID match
return true;
}
}
return false;
}
// NOTE: some additional optimizations to consider.
// 1) use the view frustum to cull those avatars that are out of view. Since avatar data doesn't need to be present
// if the avatar is not in view or in the keyhole.
// 2) after culling for view frustum, sort order the avatars by distance, send the closest ones first.
// 3) if we need to rate limit the amount of data we send, we can use a distance weighted "semi-random" function to
// determine which avatars are included in the packet stream
// 4) we should optimize the avatar data format to be more compact (100 bytes is pretty wasteful).
void broadcastAvatarData(NodeList* nodeList, sockaddr* nodeAddress) {
static unsigned char broadcastPacketBuffer[MAX_PACKET_SIZE];
static unsigned char avatarDataBuffer[MAX_PACKET_SIZE];
unsigned char* broadcastPacket = (unsigned char*)&broadcastPacketBuffer[0];
int numHeaderBytes = populateTypeAndVersion(broadcastPacket, PACKET_TYPE_BULK_AVATAR_DATA);
unsigned char* currentBufferPosition = broadcastPacket + numHeaderBytes;
int packetLength = currentBufferPosition - broadcastPacket;
int packetsSent = 0;
// send back a packet with other active node data to this node
for (NodeList::iterator node = nodeList->begin(); node != nodeList->end(); node++) {
if (node->getLinkedData() && !socketMatch(nodeAddress, node->getActiveSocket())) {
unsigned char* avatarDataEndpoint = addNodeToBroadcastPacket((unsigned char*)&avatarDataBuffer[0], &*node);
int avatarDataLength = avatarDataEndpoint - (unsigned char*)&avatarDataBuffer;
if (avatarDataLength + packetLength <= MAX_PACKET_SIZE) {
memcpy(currentBufferPosition, &avatarDataBuffer[0], avatarDataLength);
packetLength += avatarDataLength;
currentBufferPosition += avatarDataLength;
} else {
packetsSent++;
//printf("packetsSent=%d packetLength=%d\n", packetsSent, packetLength);
nodeList->getNodeSocket()->send(nodeAddress, broadcastPacket, currentBufferPosition - broadcastPacket);
// reset the packet
currentBufferPosition = broadcastPacket + numHeaderBytes;
packetLength = currentBufferPosition - broadcastPacket;
// copy the avatar that didn't fit into the next packet
memcpy(currentBufferPosition, &avatarDataBuffer[0], avatarDataLength);
packetLength += avatarDataLength;
currentBufferPosition += avatarDataLength;
}
}
}
packetsSent++;
//printf("packetsSent=%d packetLength=%d\n", packetsSent, packetLength);
nodeList->getNodeSocket()->send(nodeAddress, broadcastPacket, currentBufferPosition - broadcastPacket);
}
void *distributeVoxelsToListeners(void *args) {
NodeList* nodeList = NodeList::getInstance();
timeval lastSendTime;
while (true) {
gettimeofday(&lastSendTime, NULL);
// enumerate the nodes to send 3 packets to each
for (NodeList::iterator node = nodeList->begin(); node != nodeList->end(); node++) {
VoxelNodeData* nodeData = (VoxelNodeData*) node->getLinkedData();
// Sometimes the node data has not yet been linked, in which case we can't really do anything
if (nodeData) {
bool viewFrustumChanged = nodeData->updateCurrentViewFrustum();
if (::debugVoxelSending) {
printf("nodeData->updateCurrentViewFrustum() changed=%s\n", debug::valueOf(viewFrustumChanged));
}
deepestLevelVoxelDistributor(nodeList, node, nodeData, viewFrustumChanged);
}
}
// dynamically sleep until we need to fire off the next set of voxels
int usecToSleep = VOXEL_SEND_INTERVAL_USECS - (usecTimestampNow() - usecTimestamp(&lastSendTime));
if (usecToSleep > 0) {
usleep(usecToSleep);
} else {
if (::debugVoxelSending) {
std::cout << "Last send took too much time, not sleeping!\n";
}
}
}
pthread_exit(0);
}
int DomainServer::civetwebRequestHandler(struct mg_connection *connection) {
const struct mg_request_info* ri = mg_get_request_info(connection);
const char RESPONSE_200[] = "HTTP/1.0 200 OK\r\n\r\n";
const char RESPONSE_400[] = "HTTP/1.0 400 Bad Request\r\n\r\n";
const char URI_ASSIGNMENT[] = "/assignment";
const char URI_NODE[] = "/node";
if (strcmp(ri->request_method, "GET") == 0) {
if (strcmp(ri->uri, "/assignments.json") == 0) {
// user is asking for json list of assignments
// start with a 200 response
mg_printf(connection, "%s", RESPONSE_200);
// setup the JSON
QJsonObject assignmentJSON;
QJsonObject assignedNodesJSON;
// enumerate the NodeList to find the assigned nodes
NodeList* nodeList = NodeList::getInstance();
for (NodeList::iterator node = nodeList->begin(); node != nodeList->end(); node++) {
if (node->getLinkedData()) {
// add the node using the UUID as the key
QString uuidString = uuidStringWithoutCurlyBraces(node->getUUID());
assignedNodesJSON[uuidString] = jsonObjectForNode(&(*node));
}
}
assignmentJSON["fulfilled"] = assignedNodesJSON;
QJsonObject queuedAssignmentsJSON;
// add the queued but unfilled assignments to the json
std::deque<Assignment*>::iterator assignment = domainServerInstance->_assignmentQueue.begin();
while (assignment != domainServerInstance->_assignmentQueue.end()) {
QJsonObject queuedAssignmentJSON;
QString uuidString = uuidStringWithoutCurlyBraces((*assignment)->getUUID());
queuedAssignmentJSON[JSON_KEY_TYPE] = QString((*assignment)->getTypeName());
// if the assignment has a pool, add it
if ((*assignment)->hasPool()) {
queuedAssignmentJSON[JSON_KEY_POOL] = QString((*assignment)->getPool());
}
// add this queued assignment to the JSON
queuedAssignmentsJSON[uuidString] = queuedAssignmentJSON;
// push forward the iterator to check the next assignment
assignment++;
}
assignmentJSON["queued"] = queuedAssignmentsJSON;
// print out the created JSON
QJsonDocument assignmentDocument(assignmentJSON);
mg_printf(connection, "%s", assignmentDocument.toJson().constData());
// we've processed this request
return 1;
} else if (strcmp(ri->uri, "/nodes.json") == 0) {
// start with a 200 response
mg_printf(connection, "%s", RESPONSE_200);
// setup the JSON
QJsonObject rootJSON;
QJsonObject nodesJSON;
// enumerate the NodeList to find the assigned nodes
NodeList* nodeList = NodeList::getInstance();
for (NodeList::iterator node = nodeList->begin(); node != nodeList->end(); node++) {
// add the node using the UUID as the key
QString uuidString = uuidStringWithoutCurlyBraces(node->getUUID());
nodesJSON[uuidString] = jsonObjectForNode(&(*node));
}
rootJSON["nodes"] = nodesJSON;
// print out the created JSON
QJsonDocument nodesDocument(rootJSON);
mg_printf(connection, "%s", nodesDocument.toJson().constData());
// we've processed this request
return 1;
}
// not processed, pass to document root
return 0;
} else if (strcmp(ri->request_method, "POST") == 0) {
if (strcmp(ri->uri, URI_ASSIGNMENT) == 0) {
// return a 200
mg_printf(connection, "%s", RESPONSE_200);
// upload the file
mg_upload(connection, "/tmp");
return 1;
}
return 0;
} else if (strcmp(ri->request_method, "DELETE") == 0) {
// this is a DELETE request
// check if it is for an assignment
if (memcmp(ri->uri, URI_NODE, strlen(URI_NODE)) == 0) {
// pull the UUID from the url
QUuid deleteUUID = QUuid(QString(ri->uri + strlen(URI_NODE) + sizeof('/')));
if (!deleteUUID.isNull()) {
Node *nodeToKill = NodeList::getInstance()->nodeWithUUID(deleteUUID);
if (nodeToKill) {
// start with a 200 response
mg_printf(connection, "%s", RESPONSE_200);
// we have a valid UUID and node - kill the node that has this assignment
NodeList::getInstance()->killNode(nodeToKill);
// successfully processed request
return 1;
}
}
}
// request not processed - bad request
mg_printf(connection, "%s", RESPONSE_400);
// this was processed by civetweb
return 1;
} else {
// have mongoose process this request from the document_root
return 0;
}
}
int DomainServer::run() {
NodeList* nodeList = NodeList::getInstance();
nodeList->addHook(this);
ssize_t receivedBytes = 0;
char nodeType = '\0';
unsigned char broadcastPacket[MAX_PACKET_SIZE];
unsigned char packetData[MAX_PACKET_SIZE];
unsigned char* currentBufferPos;
unsigned char* startPointer;
sockaddr_in senderAddress, nodePublicAddress, nodeLocalAddress;
nodePublicAddress.sin_family = AF_INET;
nodeLocalAddress.sin_family = AF_INET;
nodeList->startSilentNodeRemovalThread();
if (!_staticAssignmentFile.exists() || _voxelServerConfig) {
if (_voxelServerConfig) {
// we have a new VS config, clear the existing file to start fresh
_staticAssignmentFile.remove();
}
prepopulateStaticAssignmentFile();
}
_staticAssignmentFile.open(QIODevice::ReadWrite);
_staticAssignmentFileData = _staticAssignmentFile.map(0, _staticAssignmentFile.size());
_staticAssignments = (Assignment*) _staticAssignmentFileData;
timeval startTime;
gettimeofday(&startTime, NULL);
while (true) {
while (nodeList->getNodeSocket()->receive((sockaddr *)&senderAddress, packetData, &receivedBytes) &&
packetVersionMatch(packetData)) {
if (packetData[0] == PACKET_TYPE_DOMAIN_REPORT_FOR_DUTY || packetData[0] == PACKET_TYPE_DOMAIN_LIST_REQUEST) {
// this is an RFD or domain list request packet, and there is a version match
int numBytesSenderHeader = numBytesForPacketHeader(packetData);
nodeType = *(packetData + numBytesSenderHeader);
int packetIndex = numBytesSenderHeader + sizeof(NODE_TYPE);
QUuid nodeUUID = QUuid::fromRfc4122(QByteArray(((char*) packetData + packetIndex), NUM_BYTES_RFC4122_UUID));
packetIndex += NUM_BYTES_RFC4122_UUID;
int numBytesPrivateSocket = unpackSocket(packetData + packetIndex, (sockaddr*) &nodePublicAddress);
packetIndex += numBytesPrivateSocket;
if (nodePublicAddress.sin_addr.s_addr == 0) {
// this node wants to use us its STUN server
// so set the node public address to whatever we perceive the public address to be
nodePublicAddress = senderAddress;
// if the sender is on our box then leave its public address to 0 so that
// other users attempt to reach it on the same address they have for the domain-server
if (senderAddress.sin_addr.s_addr == htonl(INADDR_LOOPBACK)) {
nodePublicAddress.sin_addr.s_addr = 0;
}
}
int numBytesPublicSocket = unpackSocket(packetData + packetIndex, (sockaddr*) &nodeLocalAddress);
packetIndex += numBytesPublicSocket;
const char STATICALLY_ASSIGNED_NODES[3] = {
NODE_TYPE_AUDIO_MIXER,
NODE_TYPE_AVATAR_MIXER,
NODE_TYPE_VOXEL_SERVER
};
Assignment* matchingStaticAssignment = NULL;
if (memchr(STATICALLY_ASSIGNED_NODES, nodeType, sizeof(STATICALLY_ASSIGNED_NODES)) == NULL
|| ((matchingStaticAssignment = matchingStaticAssignmentForCheckIn(nodeUUID, nodeType))
|| checkInWithUUIDMatchesExistingNode((sockaddr*) &nodePublicAddress,
(sockaddr*) &nodeLocalAddress,
nodeUUID)))
{
Node* checkInNode = nodeList->addOrUpdateNode(nodeUUID,
nodeType,
(sockaddr*) &nodePublicAddress,
(sockaddr*) &nodeLocalAddress);
if (matchingStaticAssignment) {
// this was a newly added node with a matching static assignment
if (_hasCompletedRestartHold) {
// remove the matching assignment from the assignment queue so we don't take the next check in
removeAssignmentFromQueue(matchingStaticAssignment);
}
// set the linked data for this node to a copy of the matching assignment
// so we can re-queue it should the node die
Assignment* nodeCopyOfMatchingAssignment = new Assignment(*matchingStaticAssignment);
checkInNode->setLinkedData(nodeCopyOfMatchingAssignment);
}
int numHeaderBytes = populateTypeAndVersion(broadcastPacket, PACKET_TYPE_DOMAIN);
currentBufferPos = broadcastPacket + numHeaderBytes;
startPointer = currentBufferPos;
unsigned char* nodeTypesOfInterest = packetData + packetIndex + sizeof(unsigned char);
int numInterestTypes = *(nodeTypesOfInterest - 1);
if (numInterestTypes > 0) {
// if the node has sent no types of interest, assume they want nothing but their own ID back
for (NodeList::iterator node = nodeList->begin(); node != nodeList->end(); node++) {
if (!node->matches((sockaddr*) &nodePublicAddress, (sockaddr*) &nodeLocalAddress, nodeType) &&
memchr(nodeTypesOfInterest, node->getType(), numInterestTypes)) {
// don't send avatar nodes to other avatars, that will come from avatar mixer
if (nodeType != NODE_TYPE_AGENT || node->getType() != NODE_TYPE_AGENT) {
currentBufferPos = addNodeToBroadcastPacket(currentBufferPos, &(*node));
}
}
}
}
// update last receive to now
uint64_t timeNow = usecTimestampNow();
checkInNode->setLastHeardMicrostamp(timeNow);
// send the constructed list back to this node
nodeList->getNodeSocket()->send((sockaddr*)&senderAddress,
broadcastPacket,
(currentBufferPos - startPointer) + numHeaderBytes);
}
} else if (packetData[0] == PACKET_TYPE_REQUEST_ASSIGNMENT) {
qDebug("Received a request for assignment.\n");
if (!_hasCompletedRestartHold) {
possiblyAddStaticAssignmentsBackToQueueAfterRestart(&startTime);
}
if (_assignmentQueue.size() > 0) {
// construct the requested assignment from the packet data
Assignment requestAssignment(packetData, receivedBytes);
Assignment* assignmentToDeploy = deployableAssignmentForRequest(requestAssignment);
if (assignmentToDeploy) {
// give this assignment out, either the type matches or the requestor said they will take any
int numHeaderBytes = populateTypeAndVersion(broadcastPacket, PACKET_TYPE_CREATE_ASSIGNMENT);
int numAssignmentBytes = assignmentToDeploy->packToBuffer(broadcastPacket + numHeaderBytes);
nodeList->getNodeSocket()->send((sockaddr*) &senderAddress,
broadcastPacket,
numHeaderBytes + numAssignmentBytes);
if (assignmentToDeploy->getNumberOfInstances() == 0) {
// there are no more instances of this script to send out, delete it
delete assignmentToDeploy;
}
}
}
} else if (packetData[0] == PACKET_TYPE_CREATE_ASSIGNMENT) {
// this is a create assignment likely recieved from a server needed more clients to help with load
// unpack it
Assignment* createAssignment = new Assignment(packetData, receivedBytes);
qDebug() << "Received a create assignment -" << *createAssignment << "\n";
// make sure we have a matching node with the UUID packed with the assignment
// if the node has sent no types of interest, assume they want nothing but their own ID back
for (NodeList::iterator node = nodeList->begin(); node != nodeList->end(); node++) {
if (node->getLinkedData()
&& socketMatch((sockaddr*) &senderAddress, node->getPublicSocket())
&& ((Assignment*) node->getLinkedData())->getUUID() == createAssignment->getUUID()) {
// give the create assignment a new UUID
createAssignment->resetUUID();
// add the assignment at the back of the queue
_assignmentQueueMutex.lock();
_assignmentQueue.push_back(createAssignment);
_assignmentQueueMutex.unlock();
// find the first available spot in the static assignments and put this assignment there
for (int i = 0; i < MAX_STATIC_ASSIGNMENT_FILE_ASSIGNMENTS; i++) {
if (_staticAssignments[i].getUUID().isNull()) {
_staticAssignments[i] = *createAssignment;
// we've stuck the assignment in, break out
break;
}
}
// we found the matching node that asked for create assignment, break out
break;
}
}
}
}
if (!_hasCompletedRestartHold) {
possiblyAddStaticAssignmentsBackToQueueAfterRestart(&startTime);
}
}
this->cleanup();
return 0;
}
