bool VoxelEditPacketSender::voxelServersExist() const {
bool hasVoxelServers = false;
bool atLeastOnJurisdictionMissing = false; // assume the best
NodeList* nodeList = NodeList::getInstance();
for (NodeList::iterator node = nodeList->begin(); node != nodeList->end(); node++) {
// only send to the NodeTypes that are NODE_TYPE_VOXEL_SERVER
if (node->getType() == NODE_TYPE_VOXEL_SERVER) {
if (nodeList->getNodeActiveSocketOrPing(&(*node))) {
QUuid nodeUUID = node->getUUID();
// If we've got Jurisdictions set, then check to see if we know the jurisdiction for this server
if (_voxelServerJurisdictions) {
// lookup our nodeUUID in the jurisdiction map, if it's missing then we're
// missing at least one jurisdiction
if ((*_voxelServerJurisdictions).find(nodeUUID) == (*_voxelServerJurisdictions).end()) {
atLeastOnJurisdictionMissing = true;
}
}
hasVoxelServers = true;
}
}
if (atLeastOnJurisdictionMissing) {
break; // no point in looking further...
}
}
return (hasVoxelServers && !atLeastOnJurisdictionMissing);
}
void VoxelEditPacketSender::queuePacketToNodes(unsigned char* buffer, ssize_t length) {
if (!_shouldSend) {
return; // bail early
}
assert(voxelServersExist()); // we must have jurisdictions to be here!!
int headerBytes = numBytesForPacketHeader(buffer) + sizeof(short) + sizeof(uint64_t);
unsigned char* octCode = buffer + headerBytes; // skip the packet header to get to the octcode
// We want to filter out edit messages for voxel servers based on the server's Jurisdiction
// But we can't really do that with a packed message, since each edit message could be destined
// for a different voxel server... So we need to actually manage multiple queued packets... one
// for each voxel server
NodeList* nodeList = NodeList::getInstance();
for (NodeList::iterator node = nodeList->begin(); node != nodeList->end(); node++) {
// only send to the NodeTypes that are NODE_TYPE_VOXEL_SERVER
if (node->getActiveSocket() != NULL && node->getType() == NODE_TYPE_VOXEL_SERVER) {
QUuid nodeUUID = node->getUUID();
bool isMyJurisdiction = true;
// we need to get the jurisdiction for this
// here we need to get the "pending packet" for this server
const JurisdictionMap& map = (*_voxelServerJurisdictions)[nodeUUID];
isMyJurisdiction = (map.isMyJurisdiction(octCode, CHECK_NODE_ONLY) == JurisdictionMap::WITHIN);
if (isMyJurisdiction) {
queuePacketToNode(nodeUUID, buffer, length);
}
}
}
}
bool DomainServer::checkInWithUUIDMatchesExistingNode(sockaddr* nodePublicSocket,
sockaddr* nodeLocalSocket,
const QUuid& checkInUUID) {
NodeList* nodeList = NodeList::getInstance();
for (NodeList::iterator node = nodeList->begin(); node != nodeList->end(); node++) {
if (node->getLinkedData()
&& socketMatch(node->getPublicSocket(), nodePublicSocket)
&& socketMatch(node->getLocalSocket(), nodeLocalSocket)
&& node->getUUID() == checkInUUID) {
// this is a matching existing node if the public socket, local socket, and UUID match
return true;
}
}
return false;
}
// NOTE: some additional optimizations to consider.
// 1) use the view frustum to cull those avatars that are out of view. Since avatar data doesn't need to be present
// if the avatar is not in view or in the keyhole.
// 2) after culling for view frustum, sort order the avatars by distance, send the closest ones first.
// 3) if we need to rate limit the amount of data we send, we can use a distance weighted "semi-random" function to
// determine which avatars are included in the packet stream
// 4) we should optimize the avatar data format to be more compact (100 bytes is pretty wasteful).
void broadcastAvatarData(NodeList* nodeList, const QUuid& receiverUUID, sockaddr* receiverAddress) {
static unsigned char broadcastPacketBuffer[MAX_PACKET_SIZE];
static unsigned char avatarDataBuffer[MAX_PACKET_SIZE];
unsigned char* broadcastPacket = (unsigned char*)&broadcastPacketBuffer[0];
int numHeaderBytes = populateTypeAndVersion(broadcastPacket, PACKET_TYPE_BULK_AVATAR_DATA);
unsigned char* currentBufferPosition = broadcastPacket + numHeaderBytes;
int packetLength = currentBufferPosition - broadcastPacket;
int packetsSent = 0;
// send back a packet with other active node data to this node
for (NodeList::iterator node = nodeList->begin(); node != nodeList->end(); node++) {
if (node->getLinkedData() && node->getUUID() != receiverUUID) {
unsigned char* avatarDataEndpoint = addNodeToBroadcastPacket((unsigned char*)&avatarDataBuffer[0], &*node);
int avatarDataLength = avatarDataEndpoint - (unsigned char*)&avatarDataBuffer;
if (avatarDataLength + packetLength <= MAX_PACKET_SIZE) {
memcpy(currentBufferPosition, &avatarDataBuffer[0], avatarDataLength);
packetLength += avatarDataLength;
currentBufferPosition += avatarDataLength;
} else {
packetsSent++;
//printf("packetsSent=%d packetLength=%d\n", packetsSent, packetLength);
nodeList->getNodeSocket()->send(receiverAddress, broadcastPacket, currentBufferPosition - broadcastPacket);
// reset the packet
currentBufferPosition = broadcastPacket + numHeaderBytes;
packetLength = currentBufferPosition - broadcastPacket;
// copy the avatar that didn't fit into the next packet
memcpy(currentBufferPosition, &avatarDataBuffer[0], avatarDataLength);
packetLength += avatarDataLength;
currentBufferPosition += avatarDataLength;
}
}
}
packetsSent++;
//printf("packetsSent=%d packetLength=%d\n", packetsSent, packetLength);
nodeList->getNodeSocket()->send(receiverAddress, broadcastPacket, currentBufferPosition - broadcastPacket);
}
// This method is called when the edit packet layer has determined that it has a fully formed packet destined for
// a known nodeID. However, we also want to handle the case where the
void VoxelEditPacketSender::queuePacketToNode(const QUuid& nodeUUID, unsigned char* buffer, ssize_t length) {
NodeList* nodeList = NodeList::getInstance();
for (NodeList::iterator node = nodeList->begin(); node != nodeList->end(); node++) {
// only send to the NodeTypes that are NODE_TYPE_VOXEL_SERVER
if (node->getType() == NODE_TYPE_VOXEL_SERVER &&
((node->getUUID() == nodeUUID) || (nodeUUID.isNull()))) {
if (nodeList->getNodeActiveSocketOrPing(&(*node))) {
sockaddr* nodeAddress = node->getActiveSocket();
queuePacketForSending(*nodeAddress, buffer, length);
// debugging output...
bool wantDebugging = false;
if (wantDebugging) {
int numBytesPacketHeader = numBytesForPacketHeader(buffer);
unsigned short int sequence = (*((unsigned short int*)(buffer + numBytesPacketHeader)));
uint64_t createdAt = (*((uint64_t*)(buffer + numBytesPacketHeader + sizeof(sequence))));
uint64_t queuedAt = usecTimestampNow();
uint64_t transitTime = queuedAt - createdAt;
const char* messageName;
switch (buffer[0]) {
case PACKET_TYPE_SET_VOXEL:
messageName = "PACKET_TYPE_SET_VOXEL";
break;
case PACKET_TYPE_SET_VOXEL_DESTRUCTIVE:
messageName = "PACKET_TYPE_SET_VOXEL_DESTRUCTIVE";
break;
case PACKET_TYPE_ERASE_VOXEL:
messageName = "PACKET_TYPE_ERASE_VOXEL";
break;
}
printf("VoxelEditPacketSender::queuePacketToNode() queued %s - command to node bytes=%ld sequence=%d transitTimeSoFar=%llu usecs\n",
messageName, length, sequence, transitTime);
}
}
}
}
}
int DomainServer::civetwebRequestHandler(struct mg_connection *connection) {
const struct mg_request_info* ri = mg_get_request_info(connection);
const char RESPONSE_200[] = "HTTP/1.0 200 OK\r\n\r\n";
const char RESPONSE_400[] = "HTTP/1.0 400 Bad Request\r\n\r\n";
const char URI_ASSIGNMENT[] = "/assignment";
const char URI_NODE[] = "/node";
if (strcmp(ri->request_method, "GET") == 0) {
if (strcmp(ri->uri, "/assignments.json") == 0) {
// user is asking for json list of assignments
// start with a 200 response
mg_printf(connection, "%s", RESPONSE_200);
// setup the JSON
QJsonObject assignmentJSON;
QJsonObject assignedNodesJSON;
// enumerate the NodeList to find the assigned nodes
NodeList* nodeList = NodeList::getInstance();
for (NodeList::iterator node = nodeList->begin(); node != nodeList->end(); node++) {
if (node->getLinkedData()) {
// add the node using the UUID as the key
QString uuidString = uuidStringWithoutCurlyBraces(node->getUUID());
assignedNodesJSON[uuidString] = jsonObjectForNode(&(*node));
}
}
assignmentJSON["fulfilled"] = assignedNodesJSON;
QJsonObject queuedAssignmentsJSON;
// add the queued but unfilled assignments to the json
std::deque<Assignment*>::iterator assignment = domainServerInstance->_assignmentQueue.begin();
while (assignment != domainServerInstance->_assignmentQueue.end()) {
QJsonObject queuedAssignmentJSON;
QString uuidString = uuidStringWithoutCurlyBraces((*assignment)->getUUID());
queuedAssignmentJSON[JSON_KEY_TYPE] = QString((*assignment)->getTypeName());
// if the assignment has a pool, add it
if ((*assignment)->hasPool()) {
queuedAssignmentJSON[JSON_KEY_POOL] = QString((*assignment)->getPool());
}
// add this queued assignment to the JSON
queuedAssignmentsJSON[uuidString] = queuedAssignmentJSON;
// push forward the iterator to check the next assignment
assignment++;
}
assignmentJSON["queued"] = queuedAssignmentsJSON;
// print out the created JSON
QJsonDocument assignmentDocument(assignmentJSON);
mg_printf(connection, "%s", assignmentDocument.toJson().constData());
// we've processed this request
return 1;
} else if (strcmp(ri->uri, "/nodes.json") == 0) {
// start with a 200 response
mg_printf(connection, "%s", RESPONSE_200);
// setup the JSON
QJsonObject rootJSON;
QJsonObject nodesJSON;
// enumerate the NodeList to find the assigned nodes
NodeList* nodeList = NodeList::getInstance();
for (NodeList::iterator node = nodeList->begin(); node != nodeList->end(); node++) {
// add the node using the UUID as the key
QString uuidString = uuidStringWithoutCurlyBraces(node->getUUID());
nodesJSON[uuidString] = jsonObjectForNode(&(*node));
}
rootJSON["nodes"] = nodesJSON;
// print out the created JSON
QJsonDocument nodesDocument(rootJSON);
mg_printf(connection, "%s", nodesDocument.toJson().constData());
// we've processed this request
return 1;
}
// not processed, pass to document root
return 0;
} else if (strcmp(ri->request_method, "POST") == 0) {
if (strcmp(ri->uri, URI_ASSIGNMENT) == 0) {
// return a 200
mg_printf(connection, "%s", RESPONSE_200);
// upload the file
mg_upload(connection, "/tmp");
return 1;
}
return 0;
} else if (strcmp(ri->request_method, "DELETE") == 0) {
// this is a DELETE request
// check if it is for an assignment
if (memcmp(ri->uri, URI_NODE, strlen(URI_NODE)) == 0) {
// pull the UUID from the url
QUuid deleteUUID = QUuid(QString(ri->uri + strlen(URI_NODE) + sizeof('/')));
if (!deleteUUID.isNull()) {
Node *nodeToKill = NodeList::getInstance()->nodeWithUUID(deleteUUID);
if (nodeToKill) {
// start with a 200 response
mg_printf(connection, "%s", RESPONSE_200);
// we have a valid UUID and node - kill the node that has this assignment
NodeList::getInstance()->killNode(nodeToKill);
// successfully processed request
return 1;
}
}
}
// request not processed - bad request
mg_printf(connection, "%s", RESPONSE_400);
// this was processed by civetweb
return 1;
} else {
// have mongoose process this request from the document_root
return 0;
}
}
// NOTE: codeColorBuffer - is JUST the octcode/color and does not contain the packet header!
void VoxelEditPacketSender::queueVoxelEditMessage(PACKET_TYPE type, unsigned char* codeColorBuffer, ssize_t length) {
if (!_shouldSend) {
return; // bail early
}
// If we don't have voxel jurisdictions, then we will simply queue up all of these packets and wait till we have
// jurisdictions for processing
if (!voxelServersExist()) {
if (_maxPendingMessages > 0) {
EditPacketBuffer* packet = new EditPacketBuffer(type, codeColorBuffer, length);
_preServerPackets.push_back(packet);
// if we've saved MORE than out max, then clear out the oldest packet...
int allPendingMessages = _preServerSingleMessagePackets.size() + _preServerPackets.size();
if (allPendingMessages > _maxPendingMessages) {
EditPacketBuffer* packet = _preServerPackets.front();
delete packet;
_preServerPackets.erase(_preServerPackets.begin());
}
}
return; // bail early
}
// We want to filter out edit messages for voxel servers based on the server's Jurisdiction
// But we can't really do that with a packed message, since each edit message could be destined
// for a different voxel server... So we need to actually manage multiple queued packets... one
// for each voxel server
NodeList* nodeList = NodeList::getInstance();
for (NodeList::iterator node = nodeList->begin(); node != nodeList->end(); node++) {
// only send to the NodeTypes that are NODE_TYPE_VOXEL_SERVER
if (node->getActiveSocket() != NULL && node->getType() == NODE_TYPE_VOXEL_SERVER) {
QUuid nodeUUID = node->getUUID();
bool isMyJurisdiction = true;
if (_voxelServerJurisdictions) {
// we need to get the jurisdiction for this
// here we need to get the "pending packet" for this server
if ((*_voxelServerJurisdictions).find(nodeUUID) != (*_voxelServerJurisdictions).end()) {
const JurisdictionMap& map = (*_voxelServerJurisdictions)[nodeUUID];
isMyJurisdiction = (map.isMyJurisdiction(codeColorBuffer, CHECK_NODE_ONLY) == JurisdictionMap::WITHIN);
} else {
isMyJurisdiction = false;
}
}
if (isMyJurisdiction) {
EditPacketBuffer& packetBuffer = _pendingEditPackets[nodeUUID];
packetBuffer._nodeUUID = nodeUUID;
// If we're switching type, then we send the last one and start over
if ((type != packetBuffer._currentType && packetBuffer._currentSize > 0) ||
(packetBuffer._currentSize + length >= _maxPacketSize)) {
releaseQueuedPacket(packetBuffer);
initializePacket(packetBuffer, type);
}
// If the buffer is empty and not correctly initialized for our type...
if (type != packetBuffer._currentType && packetBuffer._currentSize == 0) {
initializePacket(packetBuffer, type);
}
memcpy(&packetBuffer._currentBuffer[packetBuffer._currentSize], codeColorBuffer, length);
packetBuffer._currentSize += length;
}
}
}
}
void AvatarMixer::run() {
// change the logging target name while AvatarMixer is running
Logging::setTargetName(AVATAR_MIXER_LOGGING_NAME);
NodeList* nodeList = NodeList::getInstance();
nodeList->setOwnerType(NODE_TYPE_AVATAR_MIXER);
nodeList->setNodeTypesOfInterest(&NODE_TYPE_AGENT, 1);
nodeList->linkedDataCreateCallback = attachAvatarDataToNode;
nodeList->startSilentNodeRemovalThread();
sockaddr nodeAddress = {};
ssize_t receivedBytes = 0;
unsigned char packetData[MAX_PACKET_SIZE];
QUuid nodeUUID;
Node* avatarNode = NULL;
timeval lastDomainServerCheckIn = {};
while (true) {
if (NodeList::getInstance()->getNumNoReplyDomainCheckIns() == MAX_SILENT_DOMAIN_SERVER_CHECK_INS) {
break;
}
// send a check in packet to the domain server if DOMAIN_SERVER_CHECK_IN_USECS has elapsed
if (usecTimestampNow() - usecTimestamp(&lastDomainServerCheckIn) >= DOMAIN_SERVER_CHECK_IN_USECS) {
gettimeofday(&lastDomainServerCheckIn, NULL);
NodeList::getInstance()->sendDomainServerCheckIn();
}
nodeList->possiblyPingInactiveNodes();
if (nodeList->getNodeSocket()->receive(&nodeAddress, packetData, &receivedBytes) &&
packetVersionMatch(packetData)) {
switch (packetData[0]) {
case PACKET_TYPE_HEAD_DATA:
nodeUUID = QUuid::fromRfc4122(QByteArray((char*) packetData + numBytesForPacketHeader(packetData),
NUM_BYTES_RFC4122_UUID));
// add or update the node in our list
avatarNode = nodeList->nodeWithUUID(nodeUUID);
if (avatarNode) {
// parse positional data from an node
nodeList->updateNodeWithData(avatarNode, &nodeAddress, packetData, receivedBytes);
} else {
break;
}
case PACKET_TYPE_INJECT_AUDIO:
broadcastAvatarData(nodeList, nodeUUID, &nodeAddress);
break;
case PACKET_TYPE_AVATAR_URLS:
case PACKET_TYPE_AVATAR_FACE_VIDEO:
nodeUUID = QUuid::fromRfc4122(QByteArray((char*) packetData + numBytesForPacketHeader(packetData),
NUM_BYTES_RFC4122_UUID));
// let everyone else know about the update
for (NodeList::iterator node = nodeList->begin(); node != nodeList->end(); node++) {
if (node->getActiveSocket() && node->getUUID() != nodeUUID) {
nodeList->getNodeSocket()->send(node->getActiveSocket(), packetData, receivedBytes);
}
}
break;
default:
// hand this off to the NodeList
nodeList->processNodeData(&nodeAddress, packetData, receivedBytes);
break;
}
}
}
nodeList->stopSilentNodeRemovalThread();
}