// This method is called when the edit packet layer has determined that it has a fully formed packet destined for // a known nodeID. void OctreeEditPacketSender::queuePacketToNode(const QUuid& nodeUUID, unsigned char* buffer, ssize_t length) { NodeList* nodeList = NodeList::getInstance(); foreach (const SharedNodePointer& node, nodeList->getNodeHash()) { // only send to the NodeTypes that are getMyNodeType() if (node->getType() == getMyNodeType() && ((node->getUUID() == nodeUUID) || (nodeUUID.isNull()))) { if (node->getActiveSocket()) { queuePacketForSending(node, QByteArray(reinterpret_cast<char*>(buffer), length)); // debugging output... bool wantDebugging = false; if (wantDebugging) { int numBytesPacketHeader = numBytesForPacketHeader(reinterpret_cast<char*>(buffer)); unsigned short int sequence = (*((unsigned short int*)(buffer + numBytesPacketHeader))); quint64 createdAt = (*((quint64*)(buffer + numBytesPacketHeader + sizeof(sequence)))); quint64 queuedAt = usecTimestampNow(); quint64 transitTime = queuedAt - createdAt; qDebug() << "OctreeEditPacketSender::queuePacketToNode() queued " << buffer[0] << " - command to node bytes=" << length << " sequence=" << sequence << " transitTimeSoFar=" << transitTime << " usecs"; } } } } }
void OctreeHeadlessViewer::queryOctree() { char serverType = getMyNodeType(); PacketType packetType = getMyQueryMessageType(); if (_hasViewFrustum) { ConicalViewFrustums views { _viewFrustum }; _octreeQuery.setConicalViews(views); } else { _octreeQuery.clearConicalViews(); } auto nodeList = DependencyManager::get<NodeList>(); auto node = nodeList->soloNodeOfType(serverType); if (node && node->getActiveSocket()) { auto queryPacket = NLPacket::create(packetType); // encode the query data auto packetData = reinterpret_cast<unsigned char*>(queryPacket->getPayload()); int packetSize = _octreeQuery.getBroadcastData(packetData); queryPacket->setPayloadSize(packetSize); // make sure we still have an active socket nodeList->sendUnreliablePacket(*queryPacket, *node); } }
void OctreeEditPacketSender::queuePacketToNodes(unsigned char* buffer, ssize_t length) { if (!_shouldSend) { return; // bail early } assert(serversExist()); // we must have jurisdictions to be here!! int headerBytes = numBytesForPacketHeader(reinterpret_cast<char*>(buffer)) + sizeof(short) + sizeof(quint64); unsigned char* octCode = buffer + headerBytes; // skip the packet header to get to the octcode // We want to filter out edit messages for servers based on the server's Jurisdiction // But we can't really do that with a packed message, since each edit message could be destined // for a different server... So we need to actually manage multiple queued packets... one // for each server foreach (const SharedNodePointer& node, NodeList::getInstance()->getNodeHash()) { // only send to the NodeTypes that are getMyNodeType() if (node->getActiveSocket() && node->getType() == getMyNodeType()) { QUuid nodeUUID = node->getUUID(); bool isMyJurisdiction = true; // we need to get the jurisdiction for this // here we need to get the "pending packet" for this server const JurisdictionMap& map = (*_serverJurisdictions)[nodeUUID]; isMyJurisdiction = (map.isMyJurisdiction(octCode, CHECK_NODE_ONLY) == JurisdictionMap::WITHIN); if (isMyJurisdiction) { queuePacketToNode(nodeUUID, buffer, length); } } } }
bool OctreeEditPacketSender::serversExist() const { bool hasServers = false; bool atLeastOnJurisdictionMissing = false; // assume the best NodeList* nodeList = NodeList::getInstance(); foreach (const SharedNodePointer& node, nodeList->getNodeHash()) { // only send to the NodeTypes that are getMyNodeType() if (node->getType() == getMyNodeType() && node->getActiveSocket()) { QUuid nodeUUID = node->getUUID(); // If we've got Jurisdictions set, then check to see if we know the jurisdiction for this server if (_serverJurisdictions) { // lookup our nodeUUID in the jurisdiction map, if it's missing then we're // missing at least one jurisdiction if ((*_serverJurisdictions).find(nodeUUID) == (*_serverJurisdictions).end()) { atLeastOnJurisdictionMissing = true; } } hasServers = true; } if (atLeastOnJurisdictionMissing) { break; // no point in looking further... } } return (hasServers && !atLeastOnJurisdictionMissing); }
bool OctreeEditPacketSender::serversExist() const { bool hasServers = false; bool atLeastOneJurisdictionMissing = false; // assume the best DependencyManager::get<NodeList>()->eachNodeBreakable([&](const SharedNodePointer& node){ if (node->getType() == getMyNodeType() && node->getActiveSocket()) { QUuid nodeUUID = node->getUUID(); // If we've got Jurisdictions set, then check to see if we know the jurisdiction for this server if (_serverJurisdictions) { // lookup our nodeUUID in the jurisdiction map, if it's missing then we're // missing at least one jurisdiction _serverJurisdictions->lockForRead(); if ((*_serverJurisdictions).find(nodeUUID) == (*_serverJurisdictions).end()) { atLeastOneJurisdictionMissing = true; } _serverJurisdictions->unlock(); } hasServers = true; } if (atLeastOneJurisdictionMissing) { return false; // no point in looking further - return false from anonymous function } else { return true; } }); return (hasServers && !atLeastOneJurisdictionMissing); }
// This method is called when the edit packet layer has determined that it has a fully formed packet destined for // a known nodeID. void OctreeEditPacketSender::queuePacketToNode(const QUuid& nodeUUID, unsigned char* buffer, size_t length, qint64 satoshiCost) { bool wantDebug = false; DependencyManager::get<NodeList>()->eachNode([&](const SharedNodePointer& node){ // only send to the NodeTypes that are getMyNodeType() if (node->getType() == getMyNodeType() && ((node->getUUID() == nodeUUID) || (nodeUUID.isNull())) && node->getActiveSocket()) { // pack sequence number int numBytesPacketHeader = numBytesForPacketHeader(reinterpret_cast<char*>(buffer)); unsigned char* sequenceAt = buffer + numBytesPacketHeader; quint16 sequence = _outgoingSequenceNumbers[nodeUUID]++; memcpy(sequenceAt, &sequence, sizeof(quint16)); // send packet QByteArray packet(reinterpret_cast<const char*>(buffer), length); queuePacketForSending(node, packet); if (hasDestinationWalletUUID() && satoshiCost > 0) { // if we have a destination wallet UUID and a cost associated with this packet, signal that it // needs to be sent emit octreePaymentRequired(satoshiCost, nodeUUID, _destinationWalletUUID); } // add packet to history _sentPacketHistories[nodeUUID].packetSent(sequence, packet); // debugging output... if (wantDebug) { int numBytesPacketHeader = numBytesForPacketHeader(reinterpret_cast<const char*>(buffer)); unsigned short int sequence = (*((unsigned short int*)(buffer + numBytesPacketHeader))); quint64 createdAt = (*((quint64*)(buffer + numBytesPacketHeader + sizeof(sequence)))); quint64 queuedAt = usecTimestampNow(); quint64 transitTime = queuedAt - createdAt; qDebug() << "OctreeEditPacketSender::queuePacketToNode() queued " << buffer[0] << " - command to node bytes=" << length << " satoshiCost=" << satoshiCost << " sequence=" << sequence << " transitTimeSoFar=" << transitTime << " usecs"; } } }); }
// NOTE: codeColorBuffer - is JUST the octcode/color and does not contain the packet header! void OctreeEditPacketSender::queueOctreeEditMessage(PacketType type, unsigned char* codeColorBuffer, ssize_t length) { if (!_shouldSend) { return; // bail early } // If we don't have jurisdictions, then we will simply queue up all of these packets and wait till we have // jurisdictions for processing if (!serversExist()) { if (_maxPendingMessages > 0) { EditPacketBuffer* packet = new EditPacketBuffer(type, codeColorBuffer, length); _pendingPacketsLock.lock(); _preServerPackets.push_back(packet); // if we've saved MORE than out max, then clear out the oldest packet... int allPendingMessages = _preServerSingleMessagePackets.size() + _preServerPackets.size(); if (allPendingMessages > _maxPendingMessages) { EditPacketBuffer* packet = _preServerPackets.front(); delete packet; _preServerPackets.erase(_preServerPackets.begin()); } _pendingPacketsLock.unlock(); } return; // bail early } // We want to filter out edit messages for servers based on the server's Jurisdiction // But we can't really do that with a packed message, since each edit message could be destined // for a different server... So we need to actually manage multiple queued packets... one // for each server foreach (const SharedNodePointer& node, NodeList::getInstance()->getNodeHash()) { // only send to the NodeTypes that are getMyNodeType() if (node->getActiveSocket() && node->getType() == getMyNodeType()) { QUuid nodeUUID = node->getUUID(); bool isMyJurisdiction = true; if (_serverJurisdictions) { // we need to get the jurisdiction for this // here we need to get the "pending packet" for this server if ((*_serverJurisdictions).find(nodeUUID) != (*_serverJurisdictions).end()) { const JurisdictionMap& map = (*_serverJurisdictions)[nodeUUID]; isMyJurisdiction = (map.isMyJurisdiction(codeColorBuffer, CHECK_NODE_ONLY) == JurisdictionMap::WITHIN); } else { isMyJurisdiction = false; } } if (isMyJurisdiction) { EditPacketBuffer& packetBuffer = _pendingEditPackets[nodeUUID]; packetBuffer._nodeUUID = nodeUUID; // If we're switching type, then we send the last one and start over if ((type != packetBuffer._currentType && packetBuffer._currentSize > 0) || (packetBuffer._currentSize + length >= _maxPacketSize)) { releaseQueuedPacket(packetBuffer); initializePacket(packetBuffer, type); } // If the buffer is empty and not correctly initialized for our type... if (type != packetBuffer._currentType && packetBuffer._currentSize == 0) { initializePacket(packetBuffer, type); } // This is really the first time we know which server/node this particular edit message // is going to, so we couldn't adjust for clock skew till now. But here's our chance. // We call this virtual function that allows our specific type of EditPacketSender to // fixup the buffer for any clock skew if (node->getClockSkewUsec() != 0) { adjustEditPacketForClockSkew(codeColorBuffer, length, node->getClockSkewUsec()); } memcpy(&packetBuffer._currentBuffer[packetBuffer._currentSize], codeColorBuffer, length); packetBuffer._currentSize += length; } } } }
void OctreeHeadlessViewer::queryOctree() { char serverType = getMyNodeType(); PacketType packetType = getMyQueryMessageType(); NodeToJurisdictionMap& jurisdictions = *_jurisdictionListener->getJurisdictions(); bool wantExtraDebugging = false; if (wantExtraDebugging) { qCDebug(octree) << "OctreeHeadlessViewer::queryOctree() _jurisdictionListener=" << _jurisdictionListener; qCDebug(octree) << "---------------"; qCDebug(octree) << "_jurisdictionListener=" << _jurisdictionListener; qCDebug(octree) << "Jurisdictions..."; jurisdictions.lockForRead(); for (NodeToJurisdictionMapIterator i = jurisdictions.begin(); i != jurisdictions.end(); ++i) { qCDebug(octree) << i.key() << ": " << &i.value(); } jurisdictions.unlock(); qCDebug(octree) << "---------------"; } // These will be the same for all servers, so we can set them up once and then reuse for each server we send to. _octreeQuery.setWantLowResMoving(true); _octreeQuery.setWantColor(true); _octreeQuery.setWantDelta(true); _octreeQuery.setWantOcclusionCulling(false); _octreeQuery.setWantCompression(true); // TODO: should be on by default _octreeQuery.setCameraPosition(_viewFrustum.getPosition()); _octreeQuery.setCameraOrientation(_viewFrustum.getOrientation()); _octreeQuery.setCameraFov(_viewFrustum.getFieldOfView()); _octreeQuery.setCameraAspectRatio(_viewFrustum.getAspectRatio()); _octreeQuery.setCameraNearClip(_viewFrustum.getNearClip()); _octreeQuery.setCameraFarClip(_viewFrustum.getFarClip()); _octreeQuery.setCameraEyeOffsetPosition(glm::vec3()); _octreeQuery.setOctreeSizeScale(getVoxelSizeScale()); _octreeQuery.setBoundaryLevelAdjust(getBoundaryLevelAdjust()); // Iterate all of the nodes, and get a count of how many voxel servers we have... int totalServers = 0; int inViewServers = 0; int unknownJurisdictionServers = 0; DependencyManager::get<NodeList>()->eachNode([&](const SharedNodePointer& node){ // only send to the NodeTypes that are serverType if (node->getActiveSocket() && node->getType() == serverType) { totalServers++; // get the server bounds for this server QUuid nodeUUID = node->getUUID(); // if we haven't heard from this voxel server, go ahead and send it a query, so we // can get the jurisdiction... jurisdictions.lockForRead(); if (jurisdictions.find(nodeUUID) == jurisdictions.end()) { jurisdictions.unlock(); unknownJurisdictionServers++; } else { const JurisdictionMap& map = (jurisdictions)[nodeUUID]; unsigned char* rootCode = map.getRootOctalCode(); if (rootCode) { VoxelPositionSize rootDetails; voxelDetailsForCode(rootCode, rootDetails); jurisdictions.unlock(); AACube serverBounds(glm::vec3(rootDetails.x, rootDetails.y, rootDetails.z), rootDetails.s); ViewFrustum::location serverFrustumLocation = _viewFrustum.cubeInFrustum(serverBounds); if (serverFrustumLocation != ViewFrustum::OUTSIDE) { inViewServers++; } } else { jurisdictions.unlock(); } } } }); if (wantExtraDebugging) { qCDebug(octree, "Servers: total %d, in view %d, unknown jurisdiction %d", totalServers, inViewServers, unknownJurisdictionServers); } int perServerPPS = 0; const int SMALL_BUDGET = 10; int perUnknownServer = SMALL_BUDGET; int totalPPS = getMaxPacketsPerSecond(); // determine PPS based on number of servers if (inViewServers >= 1) { // set our preferred PPS to be exactly evenly divided among all of the voxel servers... and allocate 1 PPS // for each unknown jurisdiction server perServerPPS = (totalPPS / inViewServers) - (unknownJurisdictionServers * perUnknownServer); } else { if (unknownJurisdictionServers > 0) { perUnknownServer = (totalPPS / unknownJurisdictionServers); } } if (wantExtraDebugging) { qCDebug(octree, "perServerPPS: %d perUnknownServer: %d", perServerPPS, perUnknownServer); } auto nodeList = DependencyManager::get<NodeList>(); nodeList->eachNode([&](const SharedNodePointer& node){ // only send to the NodeTypes that are serverType if (node->getActiveSocket() && node->getType() == serverType) { // get the server bounds for this server QUuid nodeUUID = node->getUUID(); bool inView = false; bool unknownView = false; // if we haven't heard from this voxel server, go ahead and send it a query, so we // can get the jurisdiction... jurisdictions.lockForRead(); if (jurisdictions.find(nodeUUID) == jurisdictions.end()) { jurisdictions.unlock(); unknownView = true; // assume it's in view if (wantExtraDebugging) { qCDebug(octree) << "no known jurisdiction for node " << *node << ", assume it's visible."; } } else { const JurisdictionMap& map = (jurisdictions)[nodeUUID]; unsigned char* rootCode = map.getRootOctalCode(); if (rootCode) { VoxelPositionSize rootDetails; voxelDetailsForCode(rootCode, rootDetails); jurisdictions.unlock(); AACube serverBounds(glm::vec3(rootDetails.x, rootDetails.y, rootDetails.z), rootDetails.s); ViewFrustum::location serverFrustumLocation = _viewFrustum.cubeInFrustum(serverBounds); if (serverFrustumLocation != ViewFrustum::OUTSIDE) { inView = true; } else { inView = false; } } else { jurisdictions.unlock(); if (wantExtraDebugging) { qCDebug(octree) << "Jurisdiction without RootCode for node " << *node << ". That's unusual!"; } } } if (inView) { _octreeQuery.setMaxQueryPacketsPerSecond(perServerPPS); if (wantExtraDebugging) { qCDebug(octree) << "inView for node " << *node << ", give it budget of " << perServerPPS; } } else if (unknownView) { if (wantExtraDebugging) { qCDebug(octree) << "no known jurisdiction for node " << *node << ", give it budget of " << perUnknownServer << " to send us jurisdiction."; } // set the query's position/orientation to be degenerate in a manner that will get the scene quickly // If there's only one server, then don't do this, and just let the normal voxel query pass through // as expected... this way, we will actually get a valid scene if there is one to be seen if (totalServers > 1) { _octreeQuery.setCameraPosition(glm::vec3(-0.1,-0.1,-0.1)); const glm::quat OFF_IN_NEGATIVE_SPACE = glm::quat(-0.5, 0, -0.5, 1.0); _octreeQuery.setCameraOrientation(OFF_IN_NEGATIVE_SPACE); _octreeQuery.setCameraNearClip(0.1f); _octreeQuery.setCameraFarClip(0.1f); if (wantExtraDebugging) { qCDebug(octree) << "Using 'minimal' camera position for node" << *node; } } else { if (wantExtraDebugging) { qCDebug(octree) << "Using regular camera position for node" << *node; } } _octreeQuery.setMaxQueryPacketsPerSecond(perUnknownServer); } else { _octreeQuery.setMaxQueryPacketsPerSecond(0); } // setup the query packet auto queryPacket = NLPacket::create(packetType); _octreeQuery.getBroadcastData(reinterpret_cast<unsigned char*>(queryPacket->getPayload())); // ask the NodeList to send it nodeList->sendPacket(std::move(queryPacket), *node); } }); }
void OctreeServer::run() { // Before we do anything else, create our tree... _tree = createTree(); // change the logging target name while this is running Logging::setTargetName(getMyLoggingServerTargetName()); // Now would be a good time to parse our arguments, if we got them as assignment if (getNumPayloadBytes() > 0) { parsePayload(); } beforeRun(); // after payload has been processed qInstallMessageHandler(Logging::verboseMessageHandler); const char* STATUS_PORT = "--statusPort"; const char* statusPort = getCmdOption(_argc, _argv, STATUS_PORT); if (statusPort) { int statusPortNumber = atoi(statusPort); initMongoose(statusPortNumber); } const char* JURISDICTION_FILE = "--jurisdictionFile"; const char* jurisdictionFile = getCmdOption(_argc, _argv, JURISDICTION_FILE); if (jurisdictionFile) { qDebug("jurisdictionFile=%s\n", jurisdictionFile); qDebug("about to readFromFile().... jurisdictionFile=%s\n", jurisdictionFile); _jurisdiction = new JurisdictionMap(jurisdictionFile); qDebug("after readFromFile().... jurisdictionFile=%s\n", jurisdictionFile); } else { const char* JURISDICTION_ROOT = "--jurisdictionRoot"; const char* jurisdictionRoot = getCmdOption(_argc, _argv, JURISDICTION_ROOT); if (jurisdictionRoot) { qDebug("jurisdictionRoot=%s\n", jurisdictionRoot); } const char* JURISDICTION_ENDNODES = "--jurisdictionEndNodes"; const char* jurisdictionEndNodes = getCmdOption(_argc, _argv, JURISDICTION_ENDNODES); if (jurisdictionEndNodes) { qDebug("jurisdictionEndNodes=%s\n", jurisdictionEndNodes); } if (jurisdictionRoot || jurisdictionEndNodes) { _jurisdiction = new JurisdictionMap(jurisdictionRoot, jurisdictionEndNodes); } } NodeList* nodeList = NodeList::getInstance(); nodeList->setOwnerType(getMyNodeType()); // we need to ask the DS about agents so we can ping/reply with them const char nodeTypesOfInterest[] = { NODE_TYPE_AGENT, NODE_TYPE_ANIMATION_SERVER}; nodeList->setNodeTypesOfInterest(nodeTypesOfInterest, sizeof(nodeTypesOfInterest)); setvbuf(stdout, NULL, _IOLBF, 0); // tell our NodeList about our desire to get notifications nodeList->addHook(this); nodeList->linkedDataCreateCallback = &OctreeServer::attachQueryNodeToNode; srand((unsigned)time(0)); const char* VERBOSE_DEBUG = "--verboseDebug"; _verboseDebug = cmdOptionExists(_argc, _argv, VERBOSE_DEBUG); qDebug("verboseDebug=%s\n", debug::valueOf(_verboseDebug)); const char* DEBUG_SENDING = "--debugSending"; _debugSending = cmdOptionExists(_argc, _argv, DEBUG_SENDING); qDebug("debugSending=%s\n", debug::valueOf(_debugSending)); const char* DEBUG_RECEIVING = "--debugReceiving"; _debugReceiving = cmdOptionExists(_argc, _argv, DEBUG_RECEIVING); qDebug("debugReceiving=%s\n", debug::valueOf(_debugReceiving)); // By default we will persist, if you want to disable this, then pass in this parameter const char* NO_PERSIST = "--NoPersist"; if (cmdOptionExists(_argc, _argv, NO_PERSIST)) { _wantPersist = false; } qDebug("wantPersist=%s\n", debug::valueOf(_wantPersist)); // if we want Persistence, set up the local file and persist thread if (_wantPersist) { // Check to see if the user passed in a command line option for setting packet send rate const char* PERSIST_FILENAME = "--persistFilename"; const char* persistFilenameParameter = getCmdOption(_argc, _argv, PERSIST_FILENAME); if (persistFilenameParameter) { strcpy(_persistFilename, persistFilenameParameter); } else { strcpy(_persistFilename, getMyDefaultPersistFilename()); } qDebug("persistFilename=%s\n", _persistFilename); // now set up PersistThread _persistThread = new OctreePersistThread(_tree, _persistFilename); if (_persistThread) { _persistThread->initialize(true); } } // Debug option to demonstrate that the server's local time does not // need to be in sync with any other network node. This forces clock // skew for the individual server node const char* CLOCK_SKEW = "--clockSkew"; const char* clockSkewOption = getCmdOption(_argc, _argv, CLOCK_SKEW); if (clockSkewOption) { int clockSkew = atoi(clockSkewOption); usecTimestampNowForceClockSkew(clockSkew); qDebug("clockSkewOption=%s clockSkew=%d\n", clockSkewOption, clockSkew); } // Check to see if the user passed in a command line option for setting packet send rate const char* PACKETS_PER_SECOND = "--packetsPerSecond"; const char* packetsPerSecond = getCmdOption(_argc, _argv, PACKETS_PER_SECOND); if (packetsPerSecond) { _packetsPerClientPerInterval = atoi(packetsPerSecond) / INTERVALS_PER_SECOND; if (_packetsPerClientPerInterval < 1) { _packetsPerClientPerInterval = 1; } qDebug("packetsPerSecond=%s PACKETS_PER_CLIENT_PER_INTERVAL=%d\n", packetsPerSecond, _packetsPerClientPerInterval); } HifiSockAddr senderSockAddr; // set up our jurisdiction broadcaster... if (_jurisdiction) { _jurisdiction->setNodeType(getMyNodeType()); } _jurisdictionSender = new JurisdictionSender(_jurisdiction, getMyNodeType()); _jurisdictionSender->initialize(true); // set up our OctreeServerPacketProcessor _octreeInboundPacketProcessor = new OctreeInboundPacketProcessor(this); _octreeInboundPacketProcessor->initialize(true); // Convert now to tm struct for local timezone tm* localtm = localtime(&_started); const int MAX_TIME_LENGTH = 128; char localBuffer[MAX_TIME_LENGTH] = { 0 }; char utcBuffer[MAX_TIME_LENGTH] = { 0 }; strftime(localBuffer, MAX_TIME_LENGTH, "%m/%d/%Y %X", localtm); // Convert now to tm struct for UTC tm* gmtm = gmtime(&_started); if (gmtm != NULL) { strftime(utcBuffer, MAX_TIME_LENGTH, " [%m/%d/%Y %X UTC]", gmtm); } qDebug() << "Now running... started at: " << localBuffer << utcBuffer << "\n"; QTimer* domainServerTimer = new QTimer(this); connect(domainServerTimer, SIGNAL(timeout()), this, SLOT(checkInWithDomainServerOrExit())); domainServerTimer->start(DOMAIN_SERVER_CHECK_IN_USECS / 1000); QTimer* silentNodeTimer = new QTimer(this); connect(silentNodeTimer, SIGNAL(timeout()), nodeList, SLOT(removeSilentNodes())); silentNodeTimer->start(NODE_SILENCE_THRESHOLD_USECS / 1000); QTimer* pingNodesTimer = new QTimer(this); connect(pingNodesTimer, SIGNAL(timeout()), nodeList, SLOT(pingInactiveNodes())); pingNodesTimer->start(PING_INACTIVE_NODE_INTERVAL_USECS / 1000); }