// Version of voxel distributor that sends the deepest LOD level at once void deepestLevelVoxelDistributor(NodeList* nodeList, NodeList::iterator& node, VoxelNodeData* nodeData, bool viewFrustumChanged) { pthread_mutex_lock(&::treeLock); int truePacketsSent = 0; int trueBytesSent = 0; // FOR NOW... node tells us if it wants to receive only view frustum deltas bool wantDelta = viewFrustumChanged && nodeData->getWantDelta(); // If our packet already has content in it, then we must use the color choice of the waiting packet. // If we're starting a fresh packet, then... // If we're moving, and the client asked for low res, then we force monochrome, otherwise, use // the clients requested color state. bool wantColor = LOW_RES_MONO && nodeData->getWantLowResMoving() && viewFrustumChanged ? false : nodeData->getWantColor(); // If we have a packet waiting, and our desired want color, doesn't match the current waiting packets color // then let's just send that waiting packet. if (wantColor != nodeData->getCurrentPacketIsColor()) { if (nodeData->isPacketWaiting()) { if (::debugVoxelSending) { printf("wantColor=%s --- SENDING PARTIAL PACKET! nodeData->getCurrentPacketIsColor()=%s\n", debug::valueOf(wantColor), debug::valueOf(nodeData->getCurrentPacketIsColor())); } handlePacketSend(nodeList, node, nodeData, trueBytesSent, truePacketsSent); } else { if (::debugVoxelSending) { printf("wantColor=%s --- FIXING HEADER! nodeData->getCurrentPacketIsColor()=%s\n", debug::valueOf(wantColor), debug::valueOf(nodeData->getCurrentPacketIsColor())); } nodeData->resetVoxelPacket(); } } if (::debugVoxelSending) { printf("wantColor=%s getCurrentPacketIsColor()=%s, viewFrustumChanged=%s, getWantLowResMoving()=%s\n", debug::valueOf(wantColor), debug::valueOf(nodeData->getCurrentPacketIsColor()), debug::valueOf(viewFrustumChanged), debug::valueOf(nodeData->getWantLowResMoving())); } const ViewFrustum* lastViewFrustum = wantDelta ? &nodeData->getLastKnownViewFrustum() : NULL; if (::debugVoxelSending) { printf("deepestLevelVoxelDistributor() viewFrustumChanged=%s, nodeBag.isEmpty=%s, viewSent=%s\n", debug::valueOf(viewFrustumChanged), debug::valueOf(nodeData->nodeBag.isEmpty()), debug::valueOf(nodeData->getViewSent()) ); } // If the current view frustum has changed OR we have nothing to send, then search against // the current view frustum for things to send. if (viewFrustumChanged || nodeData->nodeBag.isEmpty()) { uint64_t now = usecTimestampNow(); if (::debugVoxelSending) { printf("(viewFrustumChanged=%s || nodeData->nodeBag.isEmpty() =%s)...\n", debug::valueOf(viewFrustumChanged), debug::valueOf(nodeData->nodeBag.isEmpty())); if (nodeData->getLastTimeBagEmpty() > 0) { float elapsedSceneSend = (now - nodeData->getLastTimeBagEmpty()) / 1000000.0f; if (viewFrustumChanged) { printf("viewFrustumChanged resetting after elapsed time to send scene = %f seconds", elapsedSceneSend); } else { printf("elapsed time to send scene = %f seconds", elapsedSceneSend); } printf(" [occlusionCulling:%s, wantDelta:%s, wantColor:%s ]\n", debug::valueOf(nodeData->getWantOcclusionCulling()), debug::valueOf(wantDelta), debug::valueOf(wantColor)); } } // if our view has changed, we need to reset these things... if (viewFrustumChanged) { nodeData->nodeBag.deleteAll(); nodeData->map.erase(); } if (!viewFrustumChanged && !nodeData->getWantDelta()) { // only set our last sent time if we weren't resetting due to frustum change uint64_t now = usecTimestampNow(); nodeData->setLastTimeBagEmpty(now); } nodeData->stats.sceneCompleted(); if (::displayVoxelStats) { nodeData->stats.printDebugDetails(); } // This is the start of "resending" the scene. nodeData->nodeBag.insert(serverTree.rootNode); // start tracking our stats bool isFullScene = (!viewFrustumChanged || !nodeData->getWantDelta()) && nodeData->getViewFrustumJustStoppedChanging(); nodeData->stats.sceneStarted(isFullScene, viewFrustumChanged, ::serverTree.rootNode); } // If we have something in our nodeBag, then turn them into packets and send them out... if (!nodeData->nodeBag.isEmpty()) { static unsigned char tempOutputBuffer[MAX_VOXEL_PACKET_SIZE - 1]; // save on allocs by making this static int bytesWritten = 0; int packetsSentThisInterval = 0; uint64_t start = usecTimestampNow(); bool shouldSendEnvironments = shouldDo(ENVIRONMENT_SEND_INTERVAL_USECS, VOXEL_SEND_INTERVAL_USECS); while (packetsSentThisInterval < PACKETS_PER_CLIENT_PER_INTERVAL - (shouldSendEnvironments ? 1 : 0)) { // Check to see if we're taking too long, and if so bail early... uint64_t now = usecTimestampNow(); long elapsedUsec = (now - start); long elapsedUsecPerPacket = (truePacketsSent == 0) ? 0 : (elapsedUsec / truePacketsSent); long usecRemaining = (VOXEL_SEND_INTERVAL_USECS - elapsedUsec); if (elapsedUsecPerPacket + SENDING_TIME_TO_SPARE > usecRemaining) { if (::debugVoxelSending) { printf("packetLoop() usecRemaining=%ld bailing early took %ld usecs to generate %d bytes in %d packets (%ld usec avg), %d nodes still to send\n", usecRemaining, elapsedUsec, trueBytesSent, truePacketsSent, elapsedUsecPerPacket, nodeData->nodeBag.count()); } break; } if (!nodeData->nodeBag.isEmpty()) { VoxelNode* subTree = nodeData->nodeBag.extract(); bool wantOcclusionCulling = nodeData->getWantOcclusionCulling(); CoverageMap* coverageMap = wantOcclusionCulling ? &nodeData->map : IGNORE_COVERAGE_MAP; int boundaryLevelAdjust = viewFrustumChanged && nodeData->getWantLowResMoving() ? LOW_RES_MOVING_ADJUST : NO_BOUNDARY_ADJUST; bool isFullScene = (!viewFrustumChanged || !nodeData->getWantDelta()) && nodeData->getViewFrustumJustStoppedChanging(); EncodeBitstreamParams params(INT_MAX, &nodeData->getCurrentViewFrustum(), wantColor, WANT_EXISTS_BITS, DONT_CHOP, wantDelta, lastViewFrustum, wantOcclusionCulling, coverageMap, boundaryLevelAdjust, nodeData->getLastTimeBagEmpty(), isFullScene, &nodeData->stats); nodeData->stats.encodeStarted(); bytesWritten = serverTree.encodeTreeBitstream(subTree, &tempOutputBuffer[0], MAX_VOXEL_PACKET_SIZE - 1, nodeData->nodeBag, params); nodeData->stats.encodeStopped(); if (nodeData->getAvailable() >= bytesWritten) { nodeData->writeToPacket(&tempOutputBuffer[0], bytesWritten); } else { handlePacketSend(nodeList, node, nodeData, trueBytesSent, truePacketsSent); packetsSentThisInterval++; nodeData->resetVoxelPacket(); nodeData->writeToPacket(&tempOutputBuffer[0], bytesWritten); } } else { if (nodeData->isPacketWaiting()) { handlePacketSend(nodeList, node, nodeData, trueBytesSent, truePacketsSent); nodeData->resetVoxelPacket(); } packetsSentThisInterval = PACKETS_PER_CLIENT_PER_INTERVAL; // done for now, no nodes left } } // send the environment packet if (shouldSendEnvironments) { int numBytesPacketHeader = populateTypeAndVersion(tempOutputBuffer, PACKET_TYPE_ENVIRONMENT_DATA); int envPacketLength = numBytesPacketHeader; for (int i = 0; i < sizeof(environmentData) / sizeof(EnvironmentData); i++) { envPacketLength += environmentData[i].getBroadcastData(tempOutputBuffer + envPacketLength); } nodeList->getNodeSocket()->send(node->getActiveSocket(), tempOutputBuffer, envPacketLength); trueBytesSent += envPacketLength; truePacketsSent++; } uint64_t end = usecTimestampNow(); int elapsedmsec = (end - start)/1000; if (elapsedmsec > 100) { if (elapsedmsec > 1000) { int elapsedsec = (end - start)/1000000; printf("WARNING! packetLoop() took %d seconds to generate %d bytes in %d packets %d nodes still to send\n", elapsedsec, trueBytesSent, truePacketsSent, nodeData->nodeBag.count()); } else { printf("WARNING! packetLoop() took %d milliseconds to generate %d bytes in %d packets, %d nodes still to send\n", elapsedmsec, trueBytesSent, truePacketsSent, nodeData->nodeBag.count()); } } else if (::debugVoxelSending) { printf("packetLoop() took %d milliseconds to generate %d bytes in %d packets, %d nodes still to send\n", elapsedmsec, trueBytesSent, truePacketsSent, nodeData->nodeBag.count()); } // if after sending packets we've emptied our bag, then we want to remember that we've sent all // the voxels from the current view frustum if (nodeData->nodeBag.isEmpty()) { nodeData->updateLastKnownViewFrustum(); nodeData->setViewSent(true); if (::debugVoxelSending) { nodeData->map.printStats(); } nodeData->map.erase(); // It would be nice if we could save this, and only reset it when the view frustum changes } } // end if bag wasn't empty, and so we sent stuff... pthread_mutex_unlock(&::treeLock); }
/// Version of voxel distributor that sends the deepest LOD level at once int OctreeSendThread::packetDistributor(const SharedNodePointer& node, OctreeQueryNode* nodeData, bool viewFrustumChanged) { int truePacketsSent = 0; int trueBytesSent = 0; int packetsSentThisInterval = 0; bool isFullScene = ((!viewFrustumChanged || !nodeData->getWantDelta()) && nodeData->getViewFrustumJustStoppedChanging()) || nodeData->hasLodChanged(); bool somethingToSend = true; // assume we have something // FOR NOW... node tells us if it wants to receive only view frustum deltas bool wantDelta = viewFrustumChanged && nodeData->getWantDelta(); // If our packet already has content in it, then we must use the color choice of the waiting packet. // If we're starting a fresh packet, then... // If we're moving, and the client asked for low res, then we force monochrome, otherwise, use // the clients requested color state. bool wantColor = nodeData->getWantColor(); bool wantCompression = nodeData->getWantCompression(); // If we have a packet waiting, and our desired want color, doesn't match the current waiting packets color // then let's just send that waiting packet. if (!nodeData->getCurrentPacketFormatMatches()) { if (nodeData->isPacketWaiting()) { packetsSentThisInterval += handlePacketSend(node, nodeData, trueBytesSent, truePacketsSent); } else { nodeData->resetOctreePacket(); } int targetSize = MAX_OCTREE_PACKET_DATA_SIZE; if (wantCompression) { targetSize = nodeData->getAvailable() - sizeof(OCTREE_PACKET_INTERNAL_SECTION_SIZE); } _packetData.changeSettings(wantCompression, targetSize); } const ViewFrustum* lastViewFrustum = wantDelta ? &nodeData->getLastKnownViewFrustum() : NULL; // If the current view frustum has changed OR we have nothing to send, then search against // the current view frustum for things to send. if (viewFrustumChanged || nodeData->nodeBag.isEmpty()) { // if our view has changed, we need to reset these things... if (viewFrustumChanged) { if (nodeData->moveShouldDump() || nodeData->hasLodChanged()) { nodeData->dumpOutOfView(); } nodeData->map.erase(); } if (!viewFrustumChanged && !nodeData->getWantDelta()) { // only set our last sent time if we weren't resetting due to frustum change quint64 now = usecTimestampNow(); nodeData->setLastTimeBagEmpty(now); } // track completed scenes and send out the stats packet accordingly nodeData->stats.sceneCompleted(); nodeData->setLastRootTimestamp(_myServer->getOctree()->getRoot()->getLastChanged()); // TODO: add these to stats page //::endSceneSleepTime = _usleepTime; //unsigned long sleepTime = ::endSceneSleepTime - ::startSceneSleepTime; //unsigned long encodeTime = nodeData->stats.getTotalEncodeTime(); //unsigned long elapsedTime = nodeData->stats.getElapsedTime(); int packetsJustSent = handlePacketSend(node, nodeData, trueBytesSent, truePacketsSent); packetsSentThisInterval += packetsJustSent; // If we're starting a full scene, then definitely we want to empty the nodeBag if (isFullScene) { nodeData->nodeBag.deleteAll(); } // TODO: add these to stats page //::startSceneSleepTime = _usleepTime; // start tracking our stats nodeData->stats.sceneStarted(isFullScene, viewFrustumChanged, _myServer->getOctree()->getRoot(), _myServer->getJurisdiction()); // This is the start of "resending" the scene. bool dontRestartSceneOnMove = false; // this is experimental if (dontRestartSceneOnMove) { if (nodeData->nodeBag.isEmpty()) { nodeData->nodeBag.insert(_myServer->getOctree()->getRoot()); // only in case of empty } } else { nodeData->nodeBag.insert(_myServer->getOctree()->getRoot()); // original behavior, reset on move or empty } } // If we have something in our nodeBag, then turn them into packets and send them out... if (!nodeData->nodeBag.isEmpty()) { int bytesWritten = 0; quint64 start = usecTimestampNow(); // TODO: add these to stats page //quint64 startCompressTimeMsecs = OctreePacketData::getCompressContentTime() / 1000; //quint64 startCompressCalls = OctreePacketData::getCompressContentCalls(); int clientMaxPacketsPerInterval = std::max(1,(nodeData->getMaxOctreePacketsPerSecond() / INTERVALS_PER_SECOND)); int maxPacketsPerInterval = std::min(clientMaxPacketsPerInterval, _myServer->getPacketsPerClientPerInterval()); int extraPackingAttempts = 0; bool completedScene = false; while (somethingToSend && packetsSentThisInterval < maxPacketsPerInterval) { float lockWaitElapsedUsec = OctreeServer::SKIP_TIME; float encodeElapsedUsec = OctreeServer::SKIP_TIME; float compressAndWriteElapsedUsec = OctreeServer::SKIP_TIME; float packetSendingElapsedUsec = OctreeServer::SKIP_TIME; quint64 startInside = usecTimestampNow(); bool lastNodeDidntFit = false; // assume each node fits if (!nodeData->nodeBag.isEmpty()) { OctreeElement* subTree = nodeData->nodeBag.extract(); /* TODO: Looking for a way to prevent locking and encoding a tree that is not // going to result in any packets being sent... // // If our node is root, and the root hasn't changed, and our view hasn't changed, // and we've already seen at least one duplicate packet, then we probably don't need // to lock the tree and encode, because the result should be that no bytes will be // encoded, and this will be a duplicate packet from the last one we sent... OctreeElement* root = _myServer->getOctree()->getRoot(); bool skipEncode = false; if ( (subTree == root) && (nodeData->getLastRootTimestamp() == root->getLastChanged()) && !viewFrustumChanged && (nodeData->getDuplicatePacketCount() > 0) ) { qDebug() << "is root, root not changed, view not changed, already seen a duplicate!" << "Can we skip it?"; skipEncode = true; } */ bool wantOcclusionCulling = nodeData->getWantOcclusionCulling(); CoverageMap* coverageMap = wantOcclusionCulling ? &nodeData->map : IGNORE_COVERAGE_MAP; float voxelSizeScale = nodeData->getOctreeSizeScale(); int boundaryLevelAdjustClient = nodeData->getBoundaryLevelAdjust(); int boundaryLevelAdjust = boundaryLevelAdjustClient + (viewFrustumChanged && nodeData->getWantLowResMoving() ? LOW_RES_MOVING_ADJUST : NO_BOUNDARY_ADJUST); EncodeBitstreamParams params(INT_MAX, &nodeData->getCurrentViewFrustum(), wantColor, WANT_EXISTS_BITS, DONT_CHOP, wantDelta, lastViewFrustum, wantOcclusionCulling, coverageMap, boundaryLevelAdjust, voxelSizeScale, nodeData->getLastTimeBagEmpty(), isFullScene, &nodeData->stats, _myServer->getJurisdiction()); // TODO: should this include the lock time or not? This stat is sent down to the client, // it seems like it may be a good idea to include the lock time as part of the encode time // are reported to client. Since you can encode without the lock nodeData->stats.encodeStarted(); quint64 lockWaitStart = usecTimestampNow(); _myServer->getOctree()->lockForRead(); quint64 lockWaitEnd = usecTimestampNow(); lockWaitElapsedUsec = (float)(lockWaitEnd - lockWaitStart); quint64 encodeStart = usecTimestampNow(); bytesWritten = _myServer->getOctree()->encodeTreeBitstream(subTree, &_packetData, nodeData->nodeBag, params); quint64 encodeEnd = usecTimestampNow(); encodeElapsedUsec = (float)(encodeEnd - encodeStart); // If after calling encodeTreeBitstream() there are no nodes left to send, then we know we've // sent the entire scene. We want to know this below so we'll actually write this content into // the packet and send it completedScene = nodeData->nodeBag.isEmpty(); // if we're trying to fill a full size packet, then we use this logic to determine if we have a DIDNT_FIT case. if (_packetData.getTargetSize() == MAX_OCTREE_PACKET_DATA_SIZE) { if (_packetData.hasContent() && bytesWritten == 0 && params.stopReason == EncodeBitstreamParams::DIDNT_FIT) { lastNodeDidntFit = true; } } else { // in compressed mode and we are trying to pack more... and we don't care if the _packetData has // content or not... because in this case even if we were unable to pack any data, we want to drop // below to our sendNow logic, but we do want to track that we attempted to pack extra extraPackingAttempts++; if (bytesWritten == 0 && params.stopReason == EncodeBitstreamParams::DIDNT_FIT) { lastNodeDidntFit = true; } } nodeData->stats.encodeStopped(); _myServer->getOctree()->unlock(); } else { // If the bag was empty then we didn't even attempt to encode, and so we know the bytesWritten were 0 bytesWritten = 0; somethingToSend = false; // this will cause us to drop out of the loop... } // If the last node didn't fit, but we're in compressed mode, then we actually want to see if we can fit a // little bit more in this packet. To do this we write into the packet, but don't send it yet, we'll // keep attempting to write in compressed mode to add more compressed segments // We only consider sending anything if there is something in the _packetData to send... But // if bytesWritten == 0 it means either the subTree couldn't fit or we had an empty bag... Both cases // mean we should send the previous packet contents and reset it. if (completedScene || lastNodeDidntFit) { if (_packetData.hasContent()) { quint64 compressAndWriteStart = usecTimestampNow(); // if for some reason the finalized size is greater than our available size, then probably the "compressed" // form actually inflated beyond our padding, and in this case we will send the current packet, then // write to out new packet... unsigned int writtenSize = _packetData.getFinalizedSize() + (nodeData->getCurrentPacketIsCompressed() ? sizeof(OCTREE_PACKET_INTERNAL_SECTION_SIZE) : 0); if (writtenSize > nodeData->getAvailable()) { packetsSentThisInterval += handlePacketSend(node, nodeData, trueBytesSent, truePacketsSent); } nodeData->writeToPacket(_packetData.getFinalizedData(), _packetData.getFinalizedSize()); extraPackingAttempts = 0; quint64 compressAndWriteEnd = usecTimestampNow(); compressAndWriteElapsedUsec = (float)(compressAndWriteEnd - compressAndWriteStart); } // If we're not running compressed, then we know we can just send now. Or if we're running compressed, but // the packet doesn't have enough space to bother attempting to pack more... bool sendNow = true; if (nodeData->getCurrentPacketIsCompressed() && nodeData->getAvailable() >= MINIMUM_ATTEMPT_MORE_PACKING && extraPackingAttempts <= REASONABLE_NUMBER_OF_PACKING_ATTEMPTS) { sendNow = false; // try to pack more } int targetSize = MAX_OCTREE_PACKET_DATA_SIZE; if (sendNow) { quint64 packetSendingStart = usecTimestampNow(); packetsSentThisInterval += handlePacketSend(node, nodeData, trueBytesSent, truePacketsSent); quint64 packetSendingEnd = usecTimestampNow(); packetSendingElapsedUsec = (float)(packetSendingEnd - packetSendingStart); if (wantCompression) { targetSize = nodeData->getAvailable() - sizeof(OCTREE_PACKET_INTERNAL_SECTION_SIZE); } } else { // If we're in compressed mode, then we want to see if we have room for more in this wire packet. // but we've finalized the _packetData, so we want to start a new section, we will do that by // resetting the packet settings with the max uncompressed size of our current available space // in the wire packet. We also include room for our section header, and a little bit of padding // to account for the fact that whenc compressing small amounts of data, we sometimes end up with // a larger compressed size then uncompressed size targetSize = nodeData->getAvailable() - sizeof(OCTREE_PACKET_INTERNAL_SECTION_SIZE) - COMPRESS_PADDING; } _packetData.changeSettings(nodeData->getWantCompression(), targetSize); // will do reset } OctreeServer::trackTreeWaitTime(lockWaitElapsedUsec); OctreeServer::trackEncodeTime(encodeElapsedUsec); OctreeServer::trackCompressAndWriteTime(compressAndWriteElapsedUsec); OctreeServer::trackPacketSendingTime(packetSendingElapsedUsec); quint64 endInside = usecTimestampNow(); quint64 elapsedInsideUsecs = endInside - startInside; OctreeServer::trackInsideTime((float)elapsedInsideUsecs); } // Here's where we can/should allow the server to send other data... // send the environment packet // TODO: should we turn this into a while loop to better handle sending multiple special packets if (_myServer->hasSpecialPacketToSend(node)) { trueBytesSent += _myServer->sendSpecialPacket(node); truePacketsSent++; packetsSentThisInterval++; } quint64 end = usecTimestampNow(); int elapsedmsec = (end - start)/USECS_PER_MSEC; OctreeServer::trackLoopTime(elapsedmsec); // TODO: add these to stats page //quint64 endCompressCalls = OctreePacketData::getCompressContentCalls(); //int elapsedCompressCalls = endCompressCalls - startCompressCalls; //quint64 endCompressTimeMsecs = OctreePacketData::getCompressContentTime() / 1000; //int elapsedCompressTimeMsecs = endCompressTimeMsecs - startCompressTimeMsecs; // if after sending packets we've emptied our bag, then we want to remember that we've sent all // the voxels from the current view frustum if (nodeData->nodeBag.isEmpty()) { nodeData->updateLastKnownViewFrustum(); nodeData->setViewSent(true); nodeData->map.erase(); // It would be nice if we could save this, and only reset it when the view frustum changes } } // end if bag wasn't empty, and so we sent stuff... return truePacketsSent; }
/// Version of voxel distributor that sends the deepest LOD level at once int OctreeSendThread::packetDistributor(const SharedNodePointer& node, OctreeQueryNode* nodeData, bool viewFrustumChanged) { bool forceDebugging = false; int truePacketsSent = 0; int trueBytesSent = 0; int packetsSentThisInterval = 0; bool somethingToSend = true; // assume we have something // FOR NOW... node tells us if it wants to receive only view frustum deltas bool wantDelta = viewFrustumChanged && nodeData->getWantDelta(); // If our packet already has content in it, then we must use the color choice of the waiting packet. // If we're starting a fresh packet, then... // If we're moving, and the client asked for low res, then we force monochrome, otherwise, use // the clients requested color state. bool wantColor = nodeData->getWantColor(); bool wantCompression = nodeData->getWantCompression(); // If we have a packet waiting, and our desired want color, doesn't match the current waiting packets color // then let's just send that waiting packet. if (!nodeData->getCurrentPacketFormatMatches()) { if (nodeData->isPacketWaiting()) { if (forceDebugging || (_myServer->wantsDebugSending() && _myServer->wantsVerboseDebug())) { qDebug("about to call handlePacketSend() .... line: %d -- format change " "wantColor=%s wantCompression=%s SENDING PARTIAL PACKET! currentPacketIsColor=%s " "currentPacketIsCompressed=%s", __LINE__, debug::valueOf(wantColor), debug::valueOf(wantCompression), debug::valueOf(nodeData->getCurrentPacketIsColor()), debug::valueOf(nodeData->getCurrentPacketIsCompressed()) ); } packetsSentThisInterval += handlePacketSend(node, nodeData, trueBytesSent, truePacketsSent); } else { if (forceDebugging || (_myServer->wantsDebugSending() && _myServer->wantsVerboseDebug())) { qDebug("wantColor=%s wantCompression=%s FIXING HEADER! currentPacketIsColor=%s currentPacketIsCompressed=%s", debug::valueOf(wantColor), debug::valueOf(wantCompression), debug::valueOf(nodeData->getCurrentPacketIsColor()), debug::valueOf(nodeData->getCurrentPacketIsCompressed()) ); } nodeData->resetOctreePacket(); } int targetSize = MAX_OCTREE_PACKET_DATA_SIZE; if (wantCompression) { targetSize = nodeData->getAvailable() - sizeof(OCTREE_PACKET_INTERNAL_SECTION_SIZE); } if (_myServer->wantsDebugSending() && _myServer->wantsVerboseDebug()) { qDebug("line:%d _packetData.changeSettings() wantCompression=%s targetSize=%d", __LINE__, debug::valueOf(wantCompression), targetSize); } _packetData.changeSettings(wantCompression, targetSize); } if (_myServer->wantsDebugSending() && _myServer->wantsVerboseDebug()) { qDebug("wantColor/isColor=%s/%s wantCompression/isCompressed=%s/%s viewFrustumChanged=%s, getWantLowResMoving()=%s", debug::valueOf(wantColor), debug::valueOf(nodeData->getCurrentPacketIsColor()), debug::valueOf(wantCompression), debug::valueOf(nodeData->getCurrentPacketIsCompressed()), debug::valueOf(viewFrustumChanged), debug::valueOf(nodeData->getWantLowResMoving())); } const ViewFrustum* lastViewFrustum = wantDelta ? &nodeData->getLastKnownViewFrustum() : NULL; if (forceDebugging || (_myServer->wantsDebugSending() && _myServer->wantsVerboseDebug())) { qDebug("packetDistributor() viewFrustumChanged=%s, nodeBag.isEmpty=%s, viewSent=%s", debug::valueOf(viewFrustumChanged), debug::valueOf(nodeData->nodeBag.isEmpty()), debug::valueOf(nodeData->getViewSent()) ); } // If the current view frustum has changed OR we have nothing to send, then search against // the current view frustum for things to send. if (viewFrustumChanged || nodeData->nodeBag.isEmpty()) { quint64 now = usecTimestampNow(); if (forceDebugging || (_myServer->wantsDebugSending() && _myServer->wantsVerboseDebug())) { qDebug("(viewFrustumChanged=%s || nodeData->nodeBag.isEmpty() =%s)...", debug::valueOf(viewFrustumChanged), debug::valueOf(nodeData->nodeBag.isEmpty())); if (nodeData->getLastTimeBagEmpty() > 0) { float elapsedSceneSend = (now - nodeData->getLastTimeBagEmpty()) / 1000000.0f; if (viewFrustumChanged) { qDebug("viewFrustumChanged resetting after elapsed time to send scene = %f seconds", elapsedSceneSend); } else { qDebug("elapsed time to send scene = %f seconds", elapsedSceneSend); } qDebug("[ occlusionCulling:%s, wantDelta:%s, wantColor:%s ]", debug::valueOf(nodeData->getWantOcclusionCulling()), debug::valueOf(wantDelta), debug::valueOf(wantColor)); } } // if our view has changed, we need to reset these things... if (viewFrustumChanged) { if (nodeData->moveShouldDump() || nodeData->hasLodChanged()) { nodeData->dumpOutOfView(); } nodeData->map.erase(); } if (!viewFrustumChanged && !nodeData->getWantDelta()) { // only set our last sent time if we weren't resetting due to frustum change quint64 now = usecTimestampNow(); nodeData->setLastTimeBagEmpty(now); } // track completed scenes and send out the stats packet accordingly nodeData->stats.sceneCompleted(); ::endSceneSleepTime = _usleepTime; unsigned long sleepTime = ::endSceneSleepTime - ::startSceneSleepTime; unsigned long encodeTime = nodeData->stats.getTotalEncodeTime(); unsigned long elapsedTime = nodeData->stats.getElapsedTime(); if (forceDebugging || (_myServer->wantsDebugSending() && _myServer->wantsVerboseDebug())) { qDebug("about to call handlePacketSend() .... line: %d -- completed scene", __LINE__ ); } int packetsJustSent = handlePacketSend(node, nodeData, trueBytesSent, truePacketsSent); packetsSentThisInterval += packetsJustSent; if (forceDebugging) { qDebug("packetsJustSent=%d packetsSentThisInterval=%d", packetsJustSent, packetsSentThisInterval); } if (forceDebugging || _myServer->wantsDebugSending()) { qDebug() << "Scene completed at " << usecTimestampNow() << "encodeTime:" << encodeTime << " sleepTime:" << sleepTime << " elapsed:" << elapsedTime << " Packets:" << _totalPackets << " Bytes:" << _totalBytes << " Wasted:" << _totalWastedBytes; } // start tracking our stats bool isFullScene = ((!viewFrustumChanged || !nodeData->getWantDelta()) && nodeData->getViewFrustumJustStoppedChanging()) || nodeData->hasLodChanged(); // If we're starting a full scene, then definitely we want to empty the nodeBag if (isFullScene) { nodeData->nodeBag.deleteAll(); } if (forceDebugging || _myServer->wantsDebugSending()) { qDebug() << "Scene started at " << usecTimestampNow() << " Packets:" << _totalPackets << " Bytes:" << _totalBytes << " Wasted:" << _totalWastedBytes; } ::startSceneSleepTime = _usleepTime; nodeData->stats.sceneStarted(isFullScene, viewFrustumChanged, _myServer->getOctree()->getRoot(), _myServer->getJurisdiction()); // This is the start of "resending" the scene. bool dontRestartSceneOnMove = false; // this is experimental if (dontRestartSceneOnMove) { if (nodeData->nodeBag.isEmpty()) { nodeData->nodeBag.insert(_myServer->getOctree()->getRoot()); // only in case of empty } } else { nodeData->nodeBag.insert(_myServer->getOctree()->getRoot()); // original behavior, reset on move or empty } } // If we have something in our nodeBag, then turn them into packets and send them out... if (!nodeData->nodeBag.isEmpty()) { int bytesWritten = 0; quint64 start = usecTimestampNow(); quint64 startCompressTimeMsecs = OctreePacketData::getCompressContentTime() / 1000; quint64 startCompressCalls = OctreePacketData::getCompressContentCalls(); int clientMaxPacketsPerInterval = std::max(1,(nodeData->getMaxOctreePacketsPerSecond() / INTERVALS_PER_SECOND)); int maxPacketsPerInterval = std::min(clientMaxPacketsPerInterval, _myServer->getPacketsPerClientPerInterval()); if (_myServer->wantsDebugSending() && _myServer->wantsVerboseDebug()) { qDebug("truePacketsSent=%d packetsSentThisInterval=%d maxPacketsPerInterval=%d server PPI=%d nodePPS=%d nodePPI=%d", truePacketsSent, packetsSentThisInterval, maxPacketsPerInterval, _myServer->getPacketsPerClientPerInterval(), nodeData->getMaxOctreePacketsPerSecond(), clientMaxPacketsPerInterval); } int extraPackingAttempts = 0; bool completedScene = false; while (somethingToSend && packetsSentThisInterval < maxPacketsPerInterval) { if (_myServer->wantsDebugSending() && _myServer->wantsVerboseDebug()) { qDebug("truePacketsSent=%d packetsSentThisInterval=%d maxPacketsPerInterval=%d server PPI=%d nodePPS=%d nodePPI=%d", truePacketsSent, packetsSentThisInterval, maxPacketsPerInterval, _myServer->getPacketsPerClientPerInterval(), nodeData->getMaxOctreePacketsPerSecond(), clientMaxPacketsPerInterval); } bool lastNodeDidntFit = false; // assume each node fits if (!nodeData->nodeBag.isEmpty()) { OctreeElement* subTree = nodeData->nodeBag.extract(); bool wantOcclusionCulling = nodeData->getWantOcclusionCulling(); CoverageMap* coverageMap = wantOcclusionCulling ? &nodeData->map : IGNORE_COVERAGE_MAP; float voxelSizeScale = nodeData->getOctreeSizeScale(); int boundaryLevelAdjustClient = nodeData->getBoundaryLevelAdjust(); int boundaryLevelAdjust = boundaryLevelAdjustClient + (viewFrustumChanged && nodeData->getWantLowResMoving() ? LOW_RES_MOVING_ADJUST : NO_BOUNDARY_ADJUST); bool isFullScene = ((!viewFrustumChanged || !nodeData->getWantDelta()) && nodeData->getViewFrustumJustStoppedChanging()) || nodeData->hasLodChanged(); EncodeBitstreamParams params(INT_MAX, &nodeData->getCurrentViewFrustum(), wantColor, WANT_EXISTS_BITS, DONT_CHOP, wantDelta, lastViewFrustum, wantOcclusionCulling, coverageMap, boundaryLevelAdjust, voxelSizeScale, nodeData->getLastTimeBagEmpty(), isFullScene, &nodeData->stats, _myServer->getJurisdiction()); _myServer->getOctree()->lockForRead(); nodeData->stats.encodeStarted(); bytesWritten = _myServer->getOctree()->encodeTreeBitstream(subTree, &_packetData, nodeData->nodeBag, params); // If after calling encodeTreeBitstream() there are no nodes left to send, then we know we've // sent the entire scene. We want to know this below so we'll actually write this content into // the packet and send it completedScene = nodeData->nodeBag.isEmpty(); // if we're trying to fill a full size packet, then we use this logic to determine if we have a DIDNT_FIT case. if (_packetData.getTargetSize() == MAX_OCTREE_PACKET_DATA_SIZE) { if (_packetData.hasContent() && bytesWritten == 0 && params.stopReason == EncodeBitstreamParams::DIDNT_FIT) { lastNodeDidntFit = true; } } else { // in compressed mode and we are trying to pack more... and we don't care if the _packetData has // content or not... because in this case even if we were unable to pack any data, we want to drop // below to our sendNow logic, but we do want to track that we attempted to pack extra extraPackingAttempts++; if (bytesWritten == 0 && params.stopReason == EncodeBitstreamParams::DIDNT_FIT) { lastNodeDidntFit = true; } } nodeData->stats.encodeStopped(); _myServer->getOctree()->unlock(); } else { // If the bag was empty then we didn't even attempt to encode, and so we know the bytesWritten were 0 bytesWritten = 0; somethingToSend = false; // this will cause us to drop out of the loop... } // If the last node didn't fit, but we're in compressed mode, then we actually want to see if we can fit a // little bit more in this packet. To do this we write into the packet, but don't send it yet, we'll // keep attempting to write in compressed mode to add more compressed segments // We only consider sending anything if there is something in the _packetData to send... But // if bytesWritten == 0 it means either the subTree couldn't fit or we had an empty bag... Both cases // mean we should send the previous packet contents and reset it. if (completedScene || lastNodeDidntFit) { if (_packetData.hasContent()) { // if for some reason the finalized size is greater than our available size, then probably the "compressed" // form actually inflated beyond our padding, and in this case we will send the current packet, then // write to out new packet... int writtenSize = _packetData.getFinalizedSize() + (nodeData->getCurrentPacketIsCompressed() ? sizeof(OCTREE_PACKET_INTERNAL_SECTION_SIZE) : 0); if (writtenSize > nodeData->getAvailable()) { if (forceDebugging || (_myServer->wantsDebugSending() && _myServer->wantsVerboseDebug())) { qDebug("about to call handlePacketSend() .... line: %d -- " "writtenSize[%d] > available[%d] too big, sending packet as is.", __LINE__, writtenSize, nodeData->getAvailable()); } packetsSentThisInterval += handlePacketSend(node, nodeData, trueBytesSent, truePacketsSent); } if (forceDebugging || (_myServer->wantsDebugSending() && _myServer->wantsVerboseDebug())) { qDebug(">>>>>> calling writeToPacket() available=%d compressedSize=%d uncompressedSize=%d target=%d", nodeData->getAvailable(), _packetData.getFinalizedSize(), _packetData.getUncompressedSize(), _packetData.getTargetSize()); } nodeData->writeToPacket(_packetData.getFinalizedData(), _packetData.getFinalizedSize()); extraPackingAttempts = 0; } // If we're not running compressed, then we know we can just send now. Or if we're running compressed, but // the packet doesn't have enough space to bother attempting to pack more... bool sendNow = true; if (nodeData->getCurrentPacketIsCompressed() && nodeData->getAvailable() >= MINIMUM_ATTEMPT_MORE_PACKING && extraPackingAttempts <= REASONABLE_NUMBER_OF_PACKING_ATTEMPTS) { sendNow = false; // try to pack more } int targetSize = MAX_OCTREE_PACKET_DATA_SIZE; if (sendNow) { if (forceDebugging) { qDebug("about to call handlePacketSend() .... line: %d -- sendNow = TRUE", __LINE__); } packetsSentThisInterval += handlePacketSend(node, nodeData, trueBytesSent, truePacketsSent); if (wantCompression) { targetSize = nodeData->getAvailable() - sizeof(OCTREE_PACKET_INTERNAL_SECTION_SIZE); } } else { // If we're in compressed mode, then we want to see if we have room for more in this wire packet. // but we've finalized the _packetData, so we want to start a new section, we will do that by // resetting the packet settings with the max uncompressed size of our current available space // in the wire packet. We also include room for our section header, and a little bit of padding // to account for the fact that whenc compressing small amounts of data, we sometimes end up with // a larger compressed size then uncompressed size targetSize = nodeData->getAvailable() - sizeof(OCTREE_PACKET_INTERNAL_SECTION_SIZE) - COMPRESS_PADDING; } if (_myServer->wantsDebugSending() && _myServer->wantsVerboseDebug()) { qDebug("line:%d _packetData.changeSettings() wantCompression=%s targetSize=%d",__LINE__, debug::valueOf(nodeData->getWantCompression()), targetSize); } _packetData.changeSettings(nodeData->getWantCompression(), targetSize); // will do reset } } // Here's where we can/should allow the server to send other data... // send the environment packet // TODO: should we turn this into a while loop to better handle sending multiple special packets if (_myServer->hasSpecialPacketToSend(node)) { trueBytesSent += _myServer->sendSpecialPacket(node); truePacketsSent++; packetsSentThisInterval++; } quint64 end = usecTimestampNow(); int elapsedmsec = (end - start)/1000; quint64 endCompressCalls = OctreePacketData::getCompressContentCalls(); int elapsedCompressCalls = endCompressCalls - startCompressCalls; quint64 endCompressTimeMsecs = OctreePacketData::getCompressContentTime() / 1000; int elapsedCompressTimeMsecs = endCompressTimeMsecs - startCompressTimeMsecs; if (elapsedmsec > 100) { if (elapsedmsec > 1000) { int elapsedsec = (end - start)/1000000; qDebug("WARNING! packetLoop() took %d seconds [%d milliseconds %d calls in compress] " "to generate %d bytes in %d packets %d nodes still to send", elapsedsec, elapsedCompressTimeMsecs, elapsedCompressCalls, trueBytesSent, truePacketsSent, nodeData->nodeBag.count()); } else { qDebug("WARNING! packetLoop() took %d milliseconds [%d milliseconds %d calls in compress] " "to generate %d bytes in %d packets, %d nodes still to send", elapsedmsec, elapsedCompressTimeMsecs, elapsedCompressCalls, trueBytesSent, truePacketsSent, nodeData->nodeBag.count()); } } else if (_myServer->wantsDebugSending() && _myServer->wantsVerboseDebug()) { qDebug("packetLoop() took %d milliseconds [%d milliseconds %d calls in compress] " "to generate %d bytes in %d packets, %d nodes still to send", elapsedmsec, elapsedCompressTimeMsecs, elapsedCompressCalls, trueBytesSent, truePacketsSent, nodeData->nodeBag.count()); } // if after sending packets we've emptied our bag, then we want to remember that we've sent all // the voxels from the current view frustum if (nodeData->nodeBag.isEmpty()) { nodeData->updateLastKnownViewFrustum(); nodeData->setViewSent(true); if (_myServer->wantsDebugSending() && _myServer->wantsVerboseDebug()) { nodeData->map.printStats(); } nodeData->map.erase(); // It would be nice if we could save this, and only reset it when the view frustum changes } if (_myServer->wantsDebugSending() && _myServer->wantsVerboseDebug()) { qDebug("truePacketsSent=%d packetsSentThisInterval=%d maxPacketsPerInterval=%d " "server PPI=%d nodePPS=%d nodePPI=%d", truePacketsSent, packetsSentThisInterval, maxPacketsPerInterval, _myServer->getPacketsPerClientPerInterval(), nodeData->getMaxOctreePacketsPerSecond(), clientMaxPacketsPerInterval); } } // end if bag wasn't empty, and so we sent stuff... return truePacketsSent; }