void AgentList::timePingReply(sockaddr *agentAddress, unsigned char *packetData) {
for(AgentList::iterator agent = begin(); agent != end(); agent++) {
if (socketMatch(agent->getPublicSocket(), agentAddress) ||
socketMatch(agent->getLocalSocket(), agentAddress)) {
int pingTime = usecTimestampNow() - *(long long *)(packetData + 1);
agent->setPingMs(pingTime / 1000);
break;
}
}
}
void NodeList::timePingReply(sockaddr *nodeAddress, unsigned char *packetData) {
for(NodeList::iterator node = begin(); node != end(); node++) {
if (socketMatch(node->getPublicSocket(), nodeAddress) ||
socketMatch(node->getLocalSocket(), nodeAddress)) {
int pingTime = usecTimestampNow() - *(uint64_t*)(packetData + numBytesForPacketHeader(packetData));
node->setPingMs(pingTime / 1000);
break;
}
}
}
void AgentList::handlePingReply(sockaddr *agentAddress) {
for(AgentList::iterator agent = begin(); agent != end(); agent++) {
// check both the public and local addresses for each agent to see if we find a match
// prioritize the private address so that we prune erroneous local matches
if (socketMatch(agent->getPublicSocket(), agentAddress)) {
agent->activatePublicSocket();
break;
} else if (socketMatch(agent->getLocalSocket(), agentAddress)) {
agent->activateLocalSocket();
break;
}
}
}
void NodeList::handlePingReply(sockaddr *nodeAddress) {
for(NodeList::iterator node = begin(); node != end(); node++) {
// check both the public and local addresses for each node to see if we find a match
// prioritize the private address so that we prune erroneous local matches
if (socketMatch(node->getPublicSocket(), nodeAddress)) {
node->activatePublicSocket();
break;
} else if (socketMatch(node->getLocalSocket(), nodeAddress)) {
node->activateLocalSocket();
break;
}
}
}
bool DomainServer::checkInWithUUIDMatchesExistingNode(sockaddr* nodePublicSocket,
sockaddr* nodeLocalSocket,
const QUuid& checkInUUID) {
NodeList* nodeList = NodeList::getInstance();
for (NodeList::iterator node = nodeList->begin(); node != nodeList->end(); node++) {
if (node->getLinkedData()
&& socketMatch(node->getPublicSocket(), nodePublicSocket)
&& socketMatch(node->getLocalSocket(), nodeLocalSocket)
&& node->getUUID() == checkInUUID) {
// this is a matching existing node if the public socket, local socket, and UUID match
return true;
}
}
return false;
}
Agent* AgentList::agentWithAddress(sockaddr *senderAddress) {
for(AgentList::iterator agent = begin(); agent != end(); agent++) {
if (agent->getActiveSocket() && socketMatch(agent->getActiveSocket(), senderAddress)) {
return &(*agent);
}
}
return NULL;
}
Node* NodeList::nodeWithAddress(sockaddr *senderAddress) {
for(NodeList::iterator node = begin(); node != end(); node++) {
if (node->getActiveSocket() && socketMatch(node->getActiveSocket(), senderAddress)) {
return &(*node);
}
}
return NULL;
}
bool DomainServer::checkInWithUUIDMatchesExistingNode(sockaddr* nodePublicSocket,
sockaddr* nodeLocalSocket,
const uchar* checkInData) {
// pull the UUID passed with the check in
QUuid checkInUUID = QUuid::fromRfc4122(QByteArray((const char*) checkInData + numBytesForPacketHeader(checkInData) +
sizeof(NODE_TYPE),
NUM_BYTES_RFC4122_UUID));
NodeList* nodeList = NodeList::getInstance();
for (NodeList::iterator node = nodeList->begin(); node != nodeList->end(); node++) {
if (node->getLinkedData()
&& socketMatch(node->getPublicSocket(), nodePublicSocket)
&& socketMatch(node->getLocalSocket(), nodeLocalSocket)
&& ((Assignment*) node->getLinkedData())->getUUID() == checkInUUID) {
// this is a matching existing node if the public socket, local socket, and UUID match
return true;
}
}
return false;
}
Node* NodeList::addOrUpdateNode(sockaddr* publicSocket, sockaddr* localSocket, char nodeType, uint16_t nodeId) {
NodeList::iterator node = end();
if (publicSocket) {
for (node = begin(); node != end(); node++) {
if (node->matches(publicSocket, localSocket, nodeType)) {
// we already have this node, stop checking
break;
}
}
}
if (node == end()) {
// if we already had this node AND it's a solo type then bust out of here
if (soloNodeOfType(nodeType)) {
return NULL;
}
// we didn't have this node, so add them
Node* newNode = new Node(publicSocket, localSocket, nodeType, nodeId);
if (socketMatch(publicSocket, localSocket)) {
// likely debugging scenario with two nodes on local network
// set the node active right away
newNode->activatePublicSocket();
}
if (newNode->getType() == NODE_TYPE_VOXEL_SERVER ||
newNode->getType() == NODE_TYPE_AVATAR_MIXER ||
newNode->getType() == NODE_TYPE_AUDIO_MIXER) {
// this is currently the cheat we use to talk directly to our test servers on EC2
// to be removed when we have a proper identification strategy
newNode->activatePublicSocket();
}
addNodeToList(newNode);
return newNode;
} else {
if (node->getType() == NODE_TYPE_AUDIO_MIXER ||
node->getType() == NODE_TYPE_VOXEL_SERVER) {
// until the Audio class also uses our nodeList, we need to update
// the lastRecvTimeUsecs for the audio mixer so it doesn't get killed and re-added continously
node->setLastHeardMicrostamp(usecTimestampNow());
}
// we had this node already, do nothing for now
return &*node;
}
}
Agent* AgentList::addOrUpdateAgent(sockaddr* publicSocket, sockaddr* localSocket, char agentType, uint16_t agentId) {
AgentList::iterator agent = end();
if (publicSocket) {
for (agent = begin(); agent != end(); agent++) {
if (agent->matches(publicSocket, localSocket, agentType)) {
// we already have this agent, stop checking
break;
}
}
}
if (agent == end()) {
// we didn't have this agent, so add them
Agent* newAgent = new Agent(publicSocket, localSocket, agentType, agentId);
if (socketMatch(publicSocket, localSocket)) {
// likely debugging scenario with two agents on local network
// set the agent active right away
newAgent->activatePublicSocket();
}
if (newAgent->getType() == AGENT_TYPE_VOXEL_SERVER ||
newAgent->getType() == AGENT_TYPE_AVATAR_MIXER ||
newAgent->getType() == AGENT_TYPE_AUDIO_MIXER) {
// this is currently the cheat we use to talk directly to our test servers on EC2
// to be removed when we have a proper identification strategy
newAgent->activatePublicSocket();
}
addAgentToList(newAgent);
return newAgent;
} else {
if (agent->getType() == AGENT_TYPE_AUDIO_MIXER ||
agent->getType() == AGENT_TYPE_VOXEL_SERVER) {
// until the Audio class also uses our agentList, we need to update
// the lastRecvTimeUsecs for the audio mixer so it doesn't get killed and re-added continously
agent->setLastHeardMicrostamp(usecTimestampNow());
}
// we had this agent already, do nothing for now
return &*agent;
}
}
// NOTE: some additional optimizations to consider.
// 1) use the view frustum to cull those avatars that are out of view. Since avatar data doesn't need to be present
// if the avatar is not in view or in the keyhole.
// 2) after culling for view frustum, sort order the avatars by distance, send the closest ones first.
// 3) if we need to rate limit the amount of data we send, we can use a distance weighted "semi-random" function to
// determine which avatars are included in the packet stream
// 4) we should optimize the avatar data format to be more compact (100 bytes is pretty wasteful).
void broadcastAvatarData(NodeList* nodeList, sockaddr* nodeAddress) {
static unsigned char broadcastPacketBuffer[MAX_PACKET_SIZE];
static unsigned char avatarDataBuffer[MAX_PACKET_SIZE];
unsigned char* broadcastPacket = (unsigned char*)&broadcastPacketBuffer[0];
int numHeaderBytes = populateTypeAndVersion(broadcastPacket, PACKET_TYPE_BULK_AVATAR_DATA);
unsigned char* currentBufferPosition = broadcastPacket + numHeaderBytes;
int packetLength = currentBufferPosition - broadcastPacket;
int packetsSent = 0;
// send back a packet with other active node data to this node
for (NodeList::iterator node = nodeList->begin(); node != nodeList->end(); node++) {
if (node->getLinkedData() && !socketMatch(nodeAddress, node->getActiveSocket())) {
unsigned char* avatarDataEndpoint = addNodeToBroadcastPacket((unsigned char*)&avatarDataBuffer[0], &*node);
int avatarDataLength = avatarDataEndpoint - (unsigned char*)&avatarDataBuffer;
if (avatarDataLength + packetLength <= MAX_PACKET_SIZE) {
memcpy(currentBufferPosition, &avatarDataBuffer[0], avatarDataLength);
packetLength += avatarDataLength;
currentBufferPosition += avatarDataLength;
} else {
packetsSent++;
//printf("packetsSent=%d packetLength=%d\n", packetsSent, packetLength);
nodeList->getNodeSocket()->send(nodeAddress, broadcastPacket, currentBufferPosition - broadcastPacket);
// reset the packet
currentBufferPosition = broadcastPacket + numHeaderBytes;
packetLength = currentBufferPosition - broadcastPacket;
// copy the avatar that didn't fit into the next packet
memcpy(currentBufferPosition, &avatarDataBuffer[0], avatarDataLength);
packetLength += avatarDataLength;
currentBufferPosition += avatarDataLength;
}
}
}
packetsSent++;
//printf("packetsSent=%d packetLength=%d\n", packetsSent, packetLength);
nodeList->getNodeSocket()->send(nodeAddress, broadcastPacket, currentBufferPosition - broadcastPacket);
}
int DomainServer::run() {
NodeList* nodeList = NodeList::getInstance();
nodeList->addHook(this);
ssize_t receivedBytes = 0;
char nodeType = '\0';
unsigned char broadcastPacket[MAX_PACKET_SIZE];
unsigned char packetData[MAX_PACKET_SIZE];
unsigned char* currentBufferPos;
unsigned char* startPointer;
sockaddr_in senderAddress, nodePublicAddress, nodeLocalAddress;
nodePublicAddress.sin_family = AF_INET;
nodeLocalAddress.sin_family = AF_INET;
nodeList->startSilentNodeRemovalThread();
if (!_staticAssignmentFile.exists() || _voxelServerConfig) {
if (_voxelServerConfig) {
// we have a new VS config, clear the existing file to start fresh
_staticAssignmentFile.remove();
}
prepopulateStaticAssignmentFile();
}
_staticAssignmentFile.open(QIODevice::ReadWrite);
_staticAssignmentFileData = _staticAssignmentFile.map(0, _staticAssignmentFile.size());
_staticAssignments = (Assignment*) _staticAssignmentFileData;
timeval startTime;
gettimeofday(&startTime, NULL);
while (true) {
while (nodeList->getNodeSocket()->receive((sockaddr *)&senderAddress, packetData, &receivedBytes) &&
packetVersionMatch(packetData)) {
if (packetData[0] == PACKET_TYPE_DOMAIN_REPORT_FOR_DUTY || packetData[0] == PACKET_TYPE_DOMAIN_LIST_REQUEST) {
// this is an RFD or domain list request packet, and there is a version match
int numBytesSenderHeader = numBytesForPacketHeader(packetData);
nodeType = *(packetData + numBytesSenderHeader);
int packetIndex = numBytesSenderHeader + sizeof(NODE_TYPE);
QUuid nodeUUID = QUuid::fromRfc4122(QByteArray(((char*) packetData + packetIndex), NUM_BYTES_RFC4122_UUID));
packetIndex += NUM_BYTES_RFC4122_UUID;
int numBytesPrivateSocket = unpackSocket(packetData + packetIndex, (sockaddr*) &nodePublicAddress);
packetIndex += numBytesPrivateSocket;
if (nodePublicAddress.sin_addr.s_addr == 0) {
// this node wants to use us its STUN server
// so set the node public address to whatever we perceive the public address to be
nodePublicAddress = senderAddress;
// if the sender is on our box then leave its public address to 0 so that
// other users attempt to reach it on the same address they have for the domain-server
if (senderAddress.sin_addr.s_addr == htonl(INADDR_LOOPBACK)) {
nodePublicAddress.sin_addr.s_addr = 0;
}
}
int numBytesPublicSocket = unpackSocket(packetData + packetIndex, (sockaddr*) &nodeLocalAddress);
packetIndex += numBytesPublicSocket;
const char STATICALLY_ASSIGNED_NODES[3] = {
NODE_TYPE_AUDIO_MIXER,
NODE_TYPE_AVATAR_MIXER,
NODE_TYPE_VOXEL_SERVER
};
Assignment* matchingStaticAssignment = NULL;
if (memchr(STATICALLY_ASSIGNED_NODES, nodeType, sizeof(STATICALLY_ASSIGNED_NODES)) == NULL
|| ((matchingStaticAssignment = matchingStaticAssignmentForCheckIn(nodeUUID, nodeType))
|| checkInWithUUIDMatchesExistingNode((sockaddr*) &nodePublicAddress,
(sockaddr*) &nodeLocalAddress,
nodeUUID)))
{
Node* checkInNode = nodeList->addOrUpdateNode(nodeUUID,
nodeType,
(sockaddr*) &nodePublicAddress,
(sockaddr*) &nodeLocalAddress);
if (matchingStaticAssignment) {
// this was a newly added node with a matching static assignment
if (_hasCompletedRestartHold) {
// remove the matching assignment from the assignment queue so we don't take the next check in
removeAssignmentFromQueue(matchingStaticAssignment);
}
// set the linked data for this node to a copy of the matching assignment
// so we can re-queue it should the node die
Assignment* nodeCopyOfMatchingAssignment = new Assignment(*matchingStaticAssignment);
checkInNode->setLinkedData(nodeCopyOfMatchingAssignment);
}
int numHeaderBytes = populateTypeAndVersion(broadcastPacket, PACKET_TYPE_DOMAIN);
currentBufferPos = broadcastPacket + numHeaderBytes;
startPointer = currentBufferPos;
unsigned char* nodeTypesOfInterest = packetData + packetIndex + sizeof(unsigned char);
int numInterestTypes = *(nodeTypesOfInterest - 1);
if (numInterestTypes > 0) {
// if the node has sent no types of interest, assume they want nothing but their own ID back
for (NodeList::iterator node = nodeList->begin(); node != nodeList->end(); node++) {
if (!node->matches((sockaddr*) &nodePublicAddress, (sockaddr*) &nodeLocalAddress, nodeType) &&
memchr(nodeTypesOfInterest, node->getType(), numInterestTypes)) {
// don't send avatar nodes to other avatars, that will come from avatar mixer
if (nodeType != NODE_TYPE_AGENT || node->getType() != NODE_TYPE_AGENT) {
currentBufferPos = addNodeToBroadcastPacket(currentBufferPos, &(*node));
}
}
}
}
// update last receive to now
uint64_t timeNow = usecTimestampNow();
checkInNode->setLastHeardMicrostamp(timeNow);
// send the constructed list back to this node
nodeList->getNodeSocket()->send((sockaddr*)&senderAddress,
broadcastPacket,
(currentBufferPos - startPointer) + numHeaderBytes);
}
} else if (packetData[0] == PACKET_TYPE_REQUEST_ASSIGNMENT) {
qDebug("Received a request for assignment.\n");
if (!_hasCompletedRestartHold) {
possiblyAddStaticAssignmentsBackToQueueAfterRestart(&startTime);
}
if (_assignmentQueue.size() > 0) {
// construct the requested assignment from the packet data
Assignment requestAssignment(packetData, receivedBytes);
Assignment* assignmentToDeploy = deployableAssignmentForRequest(requestAssignment);
if (assignmentToDeploy) {
// give this assignment out, either the type matches or the requestor said they will take any
int numHeaderBytes = populateTypeAndVersion(broadcastPacket, PACKET_TYPE_CREATE_ASSIGNMENT);
int numAssignmentBytes = assignmentToDeploy->packToBuffer(broadcastPacket + numHeaderBytes);
nodeList->getNodeSocket()->send((sockaddr*) &senderAddress,
broadcastPacket,
numHeaderBytes + numAssignmentBytes);
if (assignmentToDeploy->getNumberOfInstances() == 0) {
// there are no more instances of this script to send out, delete it
delete assignmentToDeploy;
}
}
}
} else if (packetData[0] == PACKET_TYPE_CREATE_ASSIGNMENT) {
// this is a create assignment likely recieved from a server needed more clients to help with load
// unpack it
Assignment* createAssignment = new Assignment(packetData, receivedBytes);
qDebug() << "Received a create assignment -" << *createAssignment << "\n";
// make sure we have a matching node with the UUID packed with the assignment
// if the node has sent no types of interest, assume they want nothing but their own ID back
for (NodeList::iterator node = nodeList->begin(); node != nodeList->end(); node++) {
if (node->getLinkedData()
&& socketMatch((sockaddr*) &senderAddress, node->getPublicSocket())
&& ((Assignment*) node->getLinkedData())->getUUID() == createAssignment->getUUID()) {
// give the create assignment a new UUID
createAssignment->resetUUID();
// add the assignment at the back of the queue
_assignmentQueueMutex.lock();
_assignmentQueue.push_back(createAssignment);
_assignmentQueueMutex.unlock();
// find the first available spot in the static assignments and put this assignment there
for (int i = 0; i < MAX_STATIC_ASSIGNMENT_FILE_ASSIGNMENTS; i++) {
if (_staticAssignments[i].getUUID().isNull()) {
_staticAssignments[i] = *createAssignment;
// we've stuck the assignment in, break out
break;
}
}
// we found the matching node that asked for create assignment, break out
break;
}
}
}
}
if (!_hasCompletedRestartHold) {
possiblyAddStaticAssignmentsBackToQueueAfterRestart(&startTime);
}
}
this->cleanup();
return 0;
}
bool Node::matches(sockaddr* otherPublicSocket, sockaddr* otherLocalSocket, char otherNodeType) {
// checks if two node objects are the same node (same type + local + public address)
return _type == otherNodeType
&& socketMatch(_publicSocket, otherPublicSocket)
&& socketMatch(_localSocket, otherLocalSocket);
}
