// Version of voxel distributor that sends the deepest LOD level at once
void deepestLevelVoxelDistributor(NodeList* nodeList,
NodeList::iterator& node,
VoxelNodeData* nodeData,
bool viewFrustumChanged) {
pthread_mutex_lock(&::treeLock);
int truePacketsSent = 0;
int trueBytesSent = 0;
// FOR NOW... node tells us if it wants to receive only view frustum deltas
bool wantDelta = viewFrustumChanged && nodeData->getWantDelta();
// If our packet already has content in it, then we must use the color choice of the waiting packet.
// If we're starting a fresh packet, then...
// If we're moving, and the client asked for low res, then we force monochrome, otherwise, use
// the clients requested color state.
bool wantColor = LOW_RES_MONO && nodeData->getWantLowResMoving() && viewFrustumChanged ? false : nodeData->getWantColor();
// If we have a packet waiting, and our desired want color, doesn't match the current waiting packets color
// then let's just send that waiting packet.
if (wantColor != nodeData->getCurrentPacketIsColor()) {
if (nodeData->isPacketWaiting()) {
if (::debugVoxelSending) {
printf("wantColor=%s --- SENDING PARTIAL PACKET! nodeData->getCurrentPacketIsColor()=%s\n",
debug::valueOf(wantColor), debug::valueOf(nodeData->getCurrentPacketIsColor()));
}
handlePacketSend(nodeList, node, nodeData, trueBytesSent, truePacketsSent);
} else {
if (::debugVoxelSending) {
printf("wantColor=%s --- FIXING HEADER! nodeData->getCurrentPacketIsColor()=%s\n",
debug::valueOf(wantColor), debug::valueOf(nodeData->getCurrentPacketIsColor()));
}
nodeData->resetVoxelPacket();
}
}
if (::debugVoxelSending) {
printf("wantColor=%s getCurrentPacketIsColor()=%s, viewFrustumChanged=%s, getWantLowResMoving()=%s\n",
debug::valueOf(wantColor), debug::valueOf(nodeData->getCurrentPacketIsColor()),
debug::valueOf(viewFrustumChanged), debug::valueOf(nodeData->getWantLowResMoving()));
}
const ViewFrustum* lastViewFrustum = wantDelta ? &nodeData->getLastKnownViewFrustum() : NULL;
if (::debugVoxelSending) {
printf("deepestLevelVoxelDistributor() viewFrustumChanged=%s, nodeBag.isEmpty=%s, viewSent=%s\n",
debug::valueOf(viewFrustumChanged), debug::valueOf(nodeData->nodeBag.isEmpty()),
debug::valueOf(nodeData->getViewSent())
);
}
// If the current view frustum has changed OR we have nothing to send, then search against
// the current view frustum for things to send.
if (viewFrustumChanged || nodeData->nodeBag.isEmpty()) {
uint64_t now = usecTimestampNow();
if (::debugVoxelSending) {
printf("(viewFrustumChanged=%s || nodeData->nodeBag.isEmpty() =%s)...\n",
debug::valueOf(viewFrustumChanged), debug::valueOf(nodeData->nodeBag.isEmpty()));
if (nodeData->getLastTimeBagEmpty() > 0) {
float elapsedSceneSend = (now - nodeData->getLastTimeBagEmpty()) / 1000000.0f;
if (viewFrustumChanged) {
printf("viewFrustumChanged resetting after elapsed time to send scene = %f seconds", elapsedSceneSend);
} else {
printf("elapsed time to send scene = %f seconds", elapsedSceneSend);
}
printf(" [occlusionCulling:%s, wantDelta:%s, wantColor:%s ]\n",
debug::valueOf(nodeData->getWantOcclusionCulling()), debug::valueOf(wantDelta),
debug::valueOf(wantColor));
}
}
// if our view has changed, we need to reset these things...
if (viewFrustumChanged) {
nodeData->nodeBag.deleteAll();
nodeData->map.erase();
}
if (!viewFrustumChanged && !nodeData->getWantDelta()) {
// only set our last sent time if we weren't resetting due to frustum change
uint64_t now = usecTimestampNow();
nodeData->setLastTimeBagEmpty(now);
}
nodeData->stats.sceneCompleted();
if (::displayVoxelStats) {
nodeData->stats.printDebugDetails();
}
// This is the start of "resending" the scene.
nodeData->nodeBag.insert(serverTree.rootNode);
// start tracking our stats
bool isFullScene = (!viewFrustumChanged || !nodeData->getWantDelta()) && nodeData->getViewFrustumJustStoppedChanging();
nodeData->stats.sceneStarted(isFullScene, viewFrustumChanged, ::serverTree.rootNode);
}
// If we have something in our nodeBag, then turn them into packets and send them out...
if (!nodeData->nodeBag.isEmpty()) {
static unsigned char tempOutputBuffer[MAX_VOXEL_PACKET_SIZE - 1]; // save on allocs by making this static
int bytesWritten = 0;
int packetsSentThisInterval = 0;
uint64_t start = usecTimestampNow();
bool shouldSendEnvironments = shouldDo(ENVIRONMENT_SEND_INTERVAL_USECS, VOXEL_SEND_INTERVAL_USECS);
while (packetsSentThisInterval < PACKETS_PER_CLIENT_PER_INTERVAL - (shouldSendEnvironments ? 1 : 0)) {
// Check to see if we're taking too long, and if so bail early...
uint64_t now = usecTimestampNow();
long elapsedUsec = (now - start);
long elapsedUsecPerPacket = (truePacketsSent == 0) ? 0 : (elapsedUsec / truePacketsSent);
long usecRemaining = (VOXEL_SEND_INTERVAL_USECS - elapsedUsec);
if (elapsedUsecPerPacket + SENDING_TIME_TO_SPARE > usecRemaining) {
if (::debugVoxelSending) {
printf("packetLoop() usecRemaining=%ld bailing early took %ld usecs to generate %d bytes in %d packets (%ld usec avg), %d nodes still to send\n",
usecRemaining, elapsedUsec, trueBytesSent, truePacketsSent, elapsedUsecPerPacket,
nodeData->nodeBag.count());
}
break;
}
if (!nodeData->nodeBag.isEmpty()) {
VoxelNode* subTree = nodeData->nodeBag.extract();
bool wantOcclusionCulling = nodeData->getWantOcclusionCulling();
CoverageMap* coverageMap = wantOcclusionCulling ? &nodeData->map : IGNORE_COVERAGE_MAP;
int boundaryLevelAdjust = viewFrustumChanged && nodeData->getWantLowResMoving()
? LOW_RES_MOVING_ADJUST : NO_BOUNDARY_ADJUST;
bool isFullScene = (!viewFrustumChanged || !nodeData->getWantDelta()) &&
nodeData->getViewFrustumJustStoppedChanging();
EncodeBitstreamParams params(INT_MAX, &nodeData->getCurrentViewFrustum(), wantColor,
WANT_EXISTS_BITS, DONT_CHOP, wantDelta, lastViewFrustum,
wantOcclusionCulling, coverageMap, boundaryLevelAdjust,
nodeData->getLastTimeBagEmpty(),
isFullScene, &nodeData->stats);
nodeData->stats.encodeStarted();
bytesWritten = serverTree.encodeTreeBitstream(subTree, &tempOutputBuffer[0], MAX_VOXEL_PACKET_SIZE - 1,
nodeData->nodeBag, params);
nodeData->stats.encodeStopped();
if (nodeData->getAvailable() >= bytesWritten) {
nodeData->writeToPacket(&tempOutputBuffer[0], bytesWritten);
} else {
handlePacketSend(nodeList, node, nodeData, trueBytesSent, truePacketsSent);
packetsSentThisInterval++;
nodeData->resetVoxelPacket();
nodeData->writeToPacket(&tempOutputBuffer[0], bytesWritten);
}
} else {
if (nodeData->isPacketWaiting()) {
handlePacketSend(nodeList, node, nodeData, trueBytesSent, truePacketsSent);
nodeData->resetVoxelPacket();
}
packetsSentThisInterval = PACKETS_PER_CLIENT_PER_INTERVAL; // done for now, no nodes left
}
}
// send the environment packet
if (shouldSendEnvironments) {
int numBytesPacketHeader = populateTypeAndVersion(tempOutputBuffer, PACKET_TYPE_ENVIRONMENT_DATA);
int envPacketLength = numBytesPacketHeader;
for (int i = 0; i < sizeof(environmentData) / sizeof(EnvironmentData); i++) {
envPacketLength += environmentData[i].getBroadcastData(tempOutputBuffer + envPacketLength);
}
nodeList->getNodeSocket()->send(node->getActiveSocket(), tempOutputBuffer, envPacketLength);
trueBytesSent += envPacketLength;
truePacketsSent++;
}
uint64_t end = usecTimestampNow();
int elapsedmsec = (end - start)/1000;
if (elapsedmsec > 100) {
if (elapsedmsec > 1000) {
int elapsedsec = (end - start)/1000000;
printf("WARNING! packetLoop() took %d seconds to generate %d bytes in %d packets %d nodes still to send\n",
elapsedsec, trueBytesSent, truePacketsSent, nodeData->nodeBag.count());
} else {
printf("WARNING! packetLoop() took %d milliseconds to generate %d bytes in %d packets, %d nodes still to send\n",
elapsedmsec, trueBytesSent, truePacketsSent, nodeData->nodeBag.count());
}
} else if (::debugVoxelSending) {
printf("packetLoop() took %d milliseconds to generate %d bytes in %d packets, %d nodes still to send\n",
elapsedmsec, trueBytesSent, truePacketsSent, nodeData->nodeBag.count());
}
// if after sending packets we've emptied our bag, then we want to remember that we've sent all
// the voxels from the current view frustum
if (nodeData->nodeBag.isEmpty()) {
nodeData->updateLastKnownViewFrustum();
nodeData->setViewSent(true);
if (::debugVoxelSending) {
nodeData->map.printStats();
}
nodeData->map.erase(); // It would be nice if we could save this, and only reset it when the view frustum changes
}
} // end if bag wasn't empty, and so we sent stuff...
pthread_mutex_unlock(&::treeLock);
}
/// Version of voxel distributor that sends the deepest LOD level at once
int OctreeSendThread::packetDistributor(const SharedNodePointer& node, OctreeQueryNode* nodeData, bool viewFrustumChanged) {
int truePacketsSent = 0;
int trueBytesSent = 0;
int packetsSentThisInterval = 0;
bool isFullScene = ((!viewFrustumChanged || !nodeData->getWantDelta()) && nodeData->getViewFrustumJustStoppedChanging())
|| nodeData->hasLodChanged();
bool somethingToSend = true; // assume we have something
// FOR NOW... node tells us if it wants to receive only view frustum deltas
bool wantDelta = viewFrustumChanged && nodeData->getWantDelta();
// If our packet already has content in it, then we must use the color choice of the waiting packet.
// If we're starting a fresh packet, then...
// If we're moving, and the client asked for low res, then we force monochrome, otherwise, use
// the clients requested color state.
bool wantColor = nodeData->getWantColor();
bool wantCompression = nodeData->getWantCompression();
// If we have a packet waiting, and our desired want color, doesn't match the current waiting packets color
// then let's just send that waiting packet.
if (!nodeData->getCurrentPacketFormatMatches()) {
if (nodeData->isPacketWaiting()) {
packetsSentThisInterval += handlePacketSend(node, nodeData, trueBytesSent, truePacketsSent);
} else {
nodeData->resetOctreePacket();
}
int targetSize = MAX_OCTREE_PACKET_DATA_SIZE;
if (wantCompression) {
targetSize = nodeData->getAvailable() - sizeof(OCTREE_PACKET_INTERNAL_SECTION_SIZE);
}
_packetData.changeSettings(wantCompression, targetSize);
}
const ViewFrustum* lastViewFrustum = wantDelta ? &nodeData->getLastKnownViewFrustum() : NULL;
// If the current view frustum has changed OR we have nothing to send, then search against
// the current view frustum for things to send.
if (viewFrustumChanged || nodeData->nodeBag.isEmpty()) {
// if our view has changed, we need to reset these things...
if (viewFrustumChanged) {
if (nodeData->moveShouldDump() || nodeData->hasLodChanged()) {
nodeData->dumpOutOfView();
}
nodeData->map.erase();
}
if (!viewFrustumChanged && !nodeData->getWantDelta()) {
// only set our last sent time if we weren't resetting due to frustum change
quint64 now = usecTimestampNow();
nodeData->setLastTimeBagEmpty(now);
}
// track completed scenes and send out the stats packet accordingly
nodeData->stats.sceneCompleted();
nodeData->setLastRootTimestamp(_myServer->getOctree()->getRoot()->getLastChanged());
// TODO: add these to stats page
//::endSceneSleepTime = _usleepTime;
//unsigned long sleepTime = ::endSceneSleepTime - ::startSceneSleepTime;
//unsigned long encodeTime = nodeData->stats.getTotalEncodeTime();
//unsigned long elapsedTime = nodeData->stats.getElapsedTime();
int packetsJustSent = handlePacketSend(node, nodeData, trueBytesSent, truePacketsSent);
packetsSentThisInterval += packetsJustSent;
// If we're starting a full scene, then definitely we want to empty the nodeBag
if (isFullScene) {
nodeData->nodeBag.deleteAll();
}
// TODO: add these to stats page
//::startSceneSleepTime = _usleepTime;
// start tracking our stats
nodeData->stats.sceneStarted(isFullScene, viewFrustumChanged, _myServer->getOctree()->getRoot(), _myServer->getJurisdiction());
// This is the start of "resending" the scene.
bool dontRestartSceneOnMove = false; // this is experimental
if (dontRestartSceneOnMove) {
if (nodeData->nodeBag.isEmpty()) {
nodeData->nodeBag.insert(_myServer->getOctree()->getRoot()); // only in case of empty
}
} else {
nodeData->nodeBag.insert(_myServer->getOctree()->getRoot()); // original behavior, reset on move or empty
}
}
// If we have something in our nodeBag, then turn them into packets and send them out...
if (!nodeData->nodeBag.isEmpty()) {
int bytesWritten = 0;
quint64 start = usecTimestampNow();
// TODO: add these to stats page
//quint64 startCompressTimeMsecs = OctreePacketData::getCompressContentTime() / 1000;
//quint64 startCompressCalls = OctreePacketData::getCompressContentCalls();
int clientMaxPacketsPerInterval = std::max(1,(nodeData->getMaxOctreePacketsPerSecond() / INTERVALS_PER_SECOND));
int maxPacketsPerInterval = std::min(clientMaxPacketsPerInterval, _myServer->getPacketsPerClientPerInterval());
int extraPackingAttempts = 0;
bool completedScene = false;
while (somethingToSend && packetsSentThisInterval < maxPacketsPerInterval) {
float lockWaitElapsedUsec = OctreeServer::SKIP_TIME;
float encodeElapsedUsec = OctreeServer::SKIP_TIME;
float compressAndWriteElapsedUsec = OctreeServer::SKIP_TIME;
float packetSendingElapsedUsec = OctreeServer::SKIP_TIME;
quint64 startInside = usecTimestampNow();
bool lastNodeDidntFit = false; // assume each node fits
if (!nodeData->nodeBag.isEmpty()) {
OctreeElement* subTree = nodeData->nodeBag.extract();
/* TODO: Looking for a way to prevent locking and encoding a tree that is not
// going to result in any packets being sent...
//
// If our node is root, and the root hasn't changed, and our view hasn't changed,
// and we've already seen at least one duplicate packet, then we probably don't need
// to lock the tree and encode, because the result should be that no bytes will be
// encoded, and this will be a duplicate packet from the last one we sent...
OctreeElement* root = _myServer->getOctree()->getRoot();
bool skipEncode = false;
if (
(subTree == root)
&& (nodeData->getLastRootTimestamp() == root->getLastChanged())
&& !viewFrustumChanged
&& (nodeData->getDuplicatePacketCount() > 0)
) {
qDebug() << "is root, root not changed, view not changed, already seen a duplicate!"
<< "Can we skip it?";
skipEncode = true;
}
*/
bool wantOcclusionCulling = nodeData->getWantOcclusionCulling();
CoverageMap* coverageMap = wantOcclusionCulling ? &nodeData->map : IGNORE_COVERAGE_MAP;
float voxelSizeScale = nodeData->getOctreeSizeScale();
int boundaryLevelAdjustClient = nodeData->getBoundaryLevelAdjust();
int boundaryLevelAdjust = boundaryLevelAdjustClient + (viewFrustumChanged && nodeData->getWantLowResMoving()
? LOW_RES_MOVING_ADJUST : NO_BOUNDARY_ADJUST);
EncodeBitstreamParams params(INT_MAX, &nodeData->getCurrentViewFrustum(), wantColor,
WANT_EXISTS_BITS, DONT_CHOP, wantDelta, lastViewFrustum,
wantOcclusionCulling, coverageMap, boundaryLevelAdjust, voxelSizeScale,
nodeData->getLastTimeBagEmpty(),
isFullScene, &nodeData->stats, _myServer->getJurisdiction());
// TODO: should this include the lock time or not? This stat is sent down to the client,
// it seems like it may be a good idea to include the lock time as part of the encode time
// are reported to client. Since you can encode without the lock
nodeData->stats.encodeStarted();
quint64 lockWaitStart = usecTimestampNow();
_myServer->getOctree()->lockForRead();
quint64 lockWaitEnd = usecTimestampNow();
lockWaitElapsedUsec = (float)(lockWaitEnd - lockWaitStart);
quint64 encodeStart = usecTimestampNow();
bytesWritten = _myServer->getOctree()->encodeTreeBitstream(subTree, &_packetData, nodeData->nodeBag, params);
quint64 encodeEnd = usecTimestampNow();
encodeElapsedUsec = (float)(encodeEnd - encodeStart);
// If after calling encodeTreeBitstream() there are no nodes left to send, then we know we've
// sent the entire scene. We want to know this below so we'll actually write this content into
// the packet and send it
completedScene = nodeData->nodeBag.isEmpty();
// if we're trying to fill a full size packet, then we use this logic to determine if we have a DIDNT_FIT case.
if (_packetData.getTargetSize() == MAX_OCTREE_PACKET_DATA_SIZE) {
if (_packetData.hasContent() && bytesWritten == 0 &&
params.stopReason == EncodeBitstreamParams::DIDNT_FIT) {
lastNodeDidntFit = true;
}
} else {
// in compressed mode and we are trying to pack more... and we don't care if the _packetData has
// content or not... because in this case even if we were unable to pack any data, we want to drop
// below to our sendNow logic, but we do want to track that we attempted to pack extra
extraPackingAttempts++;
if (bytesWritten == 0 && params.stopReason == EncodeBitstreamParams::DIDNT_FIT) {
lastNodeDidntFit = true;
}
}
nodeData->stats.encodeStopped();
_myServer->getOctree()->unlock();
} else {
// If the bag was empty then we didn't even attempt to encode, and so we know the bytesWritten were 0
bytesWritten = 0;
somethingToSend = false; // this will cause us to drop out of the loop...
}
// If the last node didn't fit, but we're in compressed mode, then we actually want to see if we can fit a
// little bit more in this packet. To do this we write into the packet, but don't send it yet, we'll
// keep attempting to write in compressed mode to add more compressed segments
// We only consider sending anything if there is something in the _packetData to send... But
// if bytesWritten == 0 it means either the subTree couldn't fit or we had an empty bag... Both cases
// mean we should send the previous packet contents and reset it.
if (completedScene || lastNodeDidntFit) {
if (_packetData.hasContent()) {
quint64 compressAndWriteStart = usecTimestampNow();
// if for some reason the finalized size is greater than our available size, then probably the "compressed"
// form actually inflated beyond our padding, and in this case we will send the current packet, then
// write to out new packet...
unsigned int writtenSize = _packetData.getFinalizedSize()
+ (nodeData->getCurrentPacketIsCompressed() ? sizeof(OCTREE_PACKET_INTERNAL_SECTION_SIZE) : 0);
if (writtenSize > nodeData->getAvailable()) {
packetsSentThisInterval += handlePacketSend(node, nodeData, trueBytesSent, truePacketsSent);
}
nodeData->writeToPacket(_packetData.getFinalizedData(), _packetData.getFinalizedSize());
extraPackingAttempts = 0;
quint64 compressAndWriteEnd = usecTimestampNow();
compressAndWriteElapsedUsec = (float)(compressAndWriteEnd - compressAndWriteStart);
}
// If we're not running compressed, then we know we can just send now. Or if we're running compressed, but
// the packet doesn't have enough space to bother attempting to pack more...
bool sendNow = true;
if (nodeData->getCurrentPacketIsCompressed() &&
nodeData->getAvailable() >= MINIMUM_ATTEMPT_MORE_PACKING &&
extraPackingAttempts <= REASONABLE_NUMBER_OF_PACKING_ATTEMPTS) {
sendNow = false; // try to pack more
}
int targetSize = MAX_OCTREE_PACKET_DATA_SIZE;
if (sendNow) {
quint64 packetSendingStart = usecTimestampNow();
packetsSentThisInterval += handlePacketSend(node, nodeData, trueBytesSent, truePacketsSent);
quint64 packetSendingEnd = usecTimestampNow();
packetSendingElapsedUsec = (float)(packetSendingEnd - packetSendingStart);
if (wantCompression) {
targetSize = nodeData->getAvailable() - sizeof(OCTREE_PACKET_INTERNAL_SECTION_SIZE);
}
} else {
// If we're in compressed mode, then we want to see if we have room for more in this wire packet.
// but we've finalized the _packetData, so we want to start a new section, we will do that by
// resetting the packet settings with the max uncompressed size of our current available space
// in the wire packet. We also include room for our section header, and a little bit of padding
// to account for the fact that whenc compressing small amounts of data, we sometimes end up with
// a larger compressed size then uncompressed size
targetSize = nodeData->getAvailable() - sizeof(OCTREE_PACKET_INTERNAL_SECTION_SIZE) - COMPRESS_PADDING;
}
_packetData.changeSettings(nodeData->getWantCompression(), targetSize); // will do reset
}
OctreeServer::trackTreeWaitTime(lockWaitElapsedUsec);
OctreeServer::trackEncodeTime(encodeElapsedUsec);
OctreeServer::trackCompressAndWriteTime(compressAndWriteElapsedUsec);
OctreeServer::trackPacketSendingTime(packetSendingElapsedUsec);
quint64 endInside = usecTimestampNow();
quint64 elapsedInsideUsecs = endInside - startInside;
OctreeServer::trackInsideTime((float)elapsedInsideUsecs);
}
// Here's where we can/should allow the server to send other data...
// send the environment packet
// TODO: should we turn this into a while loop to better handle sending multiple special packets
if (_myServer->hasSpecialPacketToSend(node)) {
trueBytesSent += _myServer->sendSpecialPacket(node);
truePacketsSent++;
packetsSentThisInterval++;
}
quint64 end = usecTimestampNow();
int elapsedmsec = (end - start)/USECS_PER_MSEC;
OctreeServer::trackLoopTime(elapsedmsec);
// TODO: add these to stats page
//quint64 endCompressCalls = OctreePacketData::getCompressContentCalls();
//int elapsedCompressCalls = endCompressCalls - startCompressCalls;
//quint64 endCompressTimeMsecs = OctreePacketData::getCompressContentTime() / 1000;
//int elapsedCompressTimeMsecs = endCompressTimeMsecs - startCompressTimeMsecs;
// if after sending packets we've emptied our bag, then we want to remember that we've sent all
// the voxels from the current view frustum
if (nodeData->nodeBag.isEmpty()) {
nodeData->updateLastKnownViewFrustum();
nodeData->setViewSent(true);
nodeData->map.erase(); // It would be nice if we could save this, and only reset it when the view frustum changes
}
} // end if bag wasn't empty, and so we sent stuff...
return truePacketsSent;
}
/// Version of voxel distributor that sends the deepest LOD level at once
int OctreeSendThread::packetDistributor(const SharedNodePointer& node, OctreeQueryNode* nodeData, bool viewFrustumChanged) {
bool forceDebugging = false;
int truePacketsSent = 0;
int trueBytesSent = 0;
int packetsSentThisInterval = 0;
bool somethingToSend = true; // assume we have something
// FOR NOW... node tells us if it wants to receive only view frustum deltas
bool wantDelta = viewFrustumChanged && nodeData->getWantDelta();
// If our packet already has content in it, then we must use the color choice of the waiting packet.
// If we're starting a fresh packet, then...
// If we're moving, and the client asked for low res, then we force monochrome, otherwise, use
// the clients requested color state.
bool wantColor = nodeData->getWantColor();
bool wantCompression = nodeData->getWantCompression();
// If we have a packet waiting, and our desired want color, doesn't match the current waiting packets color
// then let's just send that waiting packet.
if (!nodeData->getCurrentPacketFormatMatches()) {
if (nodeData->isPacketWaiting()) {
if (forceDebugging || (_myServer->wantsDebugSending() && _myServer->wantsVerboseDebug())) {
qDebug("about to call handlePacketSend() .... line: %d -- format change "
"wantColor=%s wantCompression=%s SENDING PARTIAL PACKET! currentPacketIsColor=%s "
"currentPacketIsCompressed=%s",
__LINE__,
debug::valueOf(wantColor), debug::valueOf(wantCompression),
debug::valueOf(nodeData->getCurrentPacketIsColor()),
debug::valueOf(nodeData->getCurrentPacketIsCompressed()) );
}
packetsSentThisInterval += handlePacketSend(node, nodeData, trueBytesSent, truePacketsSent);
} else {
if (forceDebugging || (_myServer->wantsDebugSending() && _myServer->wantsVerboseDebug())) {
qDebug("wantColor=%s wantCompression=%s FIXING HEADER! currentPacketIsColor=%s currentPacketIsCompressed=%s",
debug::valueOf(wantColor), debug::valueOf(wantCompression),
debug::valueOf(nodeData->getCurrentPacketIsColor()),
debug::valueOf(nodeData->getCurrentPacketIsCompressed()) );
}
nodeData->resetOctreePacket();
}
int targetSize = MAX_OCTREE_PACKET_DATA_SIZE;
if (wantCompression) {
targetSize = nodeData->getAvailable() - sizeof(OCTREE_PACKET_INTERNAL_SECTION_SIZE);
}
if (_myServer->wantsDebugSending() && _myServer->wantsVerboseDebug()) {
qDebug("line:%d _packetData.changeSettings() wantCompression=%s targetSize=%d", __LINE__,
debug::valueOf(wantCompression), targetSize);
}
_packetData.changeSettings(wantCompression, targetSize);
}
if (_myServer->wantsDebugSending() && _myServer->wantsVerboseDebug()) {
qDebug("wantColor/isColor=%s/%s wantCompression/isCompressed=%s/%s viewFrustumChanged=%s, getWantLowResMoving()=%s",
debug::valueOf(wantColor), debug::valueOf(nodeData->getCurrentPacketIsColor()),
debug::valueOf(wantCompression), debug::valueOf(nodeData->getCurrentPacketIsCompressed()),
debug::valueOf(viewFrustumChanged), debug::valueOf(nodeData->getWantLowResMoving()));
}
const ViewFrustum* lastViewFrustum = wantDelta ? &nodeData->getLastKnownViewFrustum() : NULL;
if (forceDebugging || (_myServer->wantsDebugSending() && _myServer->wantsVerboseDebug())) {
qDebug("packetDistributor() viewFrustumChanged=%s, nodeBag.isEmpty=%s, viewSent=%s",
debug::valueOf(viewFrustumChanged), debug::valueOf(nodeData->nodeBag.isEmpty()),
debug::valueOf(nodeData->getViewSent())
);
}
// If the current view frustum has changed OR we have nothing to send, then search against
// the current view frustum for things to send.
if (viewFrustumChanged || nodeData->nodeBag.isEmpty()) {
quint64 now = usecTimestampNow();
if (forceDebugging || (_myServer->wantsDebugSending() && _myServer->wantsVerboseDebug())) {
qDebug("(viewFrustumChanged=%s || nodeData->nodeBag.isEmpty() =%s)...",
debug::valueOf(viewFrustumChanged), debug::valueOf(nodeData->nodeBag.isEmpty()));
if (nodeData->getLastTimeBagEmpty() > 0) {
float elapsedSceneSend = (now - nodeData->getLastTimeBagEmpty()) / 1000000.0f;
if (viewFrustumChanged) {
qDebug("viewFrustumChanged resetting after elapsed time to send scene = %f seconds", elapsedSceneSend);
} else {
qDebug("elapsed time to send scene = %f seconds", elapsedSceneSend);
}
qDebug("[ occlusionCulling:%s, wantDelta:%s, wantColor:%s ]",
debug::valueOf(nodeData->getWantOcclusionCulling()), debug::valueOf(wantDelta),
debug::valueOf(wantColor));
}
}
// if our view has changed, we need to reset these things...
if (viewFrustumChanged) {
if (nodeData->moveShouldDump() || nodeData->hasLodChanged()) {
nodeData->dumpOutOfView();
}
nodeData->map.erase();
}
if (!viewFrustumChanged && !nodeData->getWantDelta()) {
// only set our last sent time if we weren't resetting due to frustum change
quint64 now = usecTimestampNow();
nodeData->setLastTimeBagEmpty(now);
}
// track completed scenes and send out the stats packet accordingly
nodeData->stats.sceneCompleted();
::endSceneSleepTime = _usleepTime;
unsigned long sleepTime = ::endSceneSleepTime - ::startSceneSleepTime;
unsigned long encodeTime = nodeData->stats.getTotalEncodeTime();
unsigned long elapsedTime = nodeData->stats.getElapsedTime();
if (forceDebugging || (_myServer->wantsDebugSending() && _myServer->wantsVerboseDebug())) {
qDebug("about to call handlePacketSend() .... line: %d -- completed scene", __LINE__ );
}
int packetsJustSent = handlePacketSend(node, nodeData, trueBytesSent, truePacketsSent);
packetsSentThisInterval += packetsJustSent;
if (forceDebugging) {
qDebug("packetsJustSent=%d packetsSentThisInterval=%d", packetsJustSent, packetsSentThisInterval);
}
if (forceDebugging || _myServer->wantsDebugSending()) {
qDebug() << "Scene completed at " << usecTimestampNow()
<< "encodeTime:" << encodeTime
<< " sleepTime:" << sleepTime
<< " elapsed:" << elapsedTime
<< " Packets:" << _totalPackets
<< " Bytes:" << _totalBytes
<< " Wasted:" << _totalWastedBytes;
}
// start tracking our stats
bool isFullScene = ((!viewFrustumChanged || !nodeData->getWantDelta())
&& nodeData->getViewFrustumJustStoppedChanging()) || nodeData->hasLodChanged();
// If we're starting a full scene, then definitely we want to empty the nodeBag
if (isFullScene) {
nodeData->nodeBag.deleteAll();
}
if (forceDebugging || _myServer->wantsDebugSending()) {
qDebug() << "Scene started at " << usecTimestampNow()
<< " Packets:" << _totalPackets
<< " Bytes:" << _totalBytes
<< " Wasted:" << _totalWastedBytes;
}
::startSceneSleepTime = _usleepTime;
nodeData->stats.sceneStarted(isFullScene, viewFrustumChanged, _myServer->getOctree()->getRoot(), _myServer->getJurisdiction());
// This is the start of "resending" the scene.
bool dontRestartSceneOnMove = false; // this is experimental
if (dontRestartSceneOnMove) {
if (nodeData->nodeBag.isEmpty()) {
nodeData->nodeBag.insert(_myServer->getOctree()->getRoot()); // only in case of empty
}
} else {
nodeData->nodeBag.insert(_myServer->getOctree()->getRoot()); // original behavior, reset on move or empty
}
}
// If we have something in our nodeBag, then turn them into packets and send them out...
if (!nodeData->nodeBag.isEmpty()) {
int bytesWritten = 0;
quint64 start = usecTimestampNow();
quint64 startCompressTimeMsecs = OctreePacketData::getCompressContentTime() / 1000;
quint64 startCompressCalls = OctreePacketData::getCompressContentCalls();
int clientMaxPacketsPerInterval = std::max(1,(nodeData->getMaxOctreePacketsPerSecond() / INTERVALS_PER_SECOND));
int maxPacketsPerInterval = std::min(clientMaxPacketsPerInterval, _myServer->getPacketsPerClientPerInterval());
if (_myServer->wantsDebugSending() && _myServer->wantsVerboseDebug()) {
qDebug("truePacketsSent=%d packetsSentThisInterval=%d maxPacketsPerInterval=%d server PPI=%d nodePPS=%d nodePPI=%d",
truePacketsSent, packetsSentThisInterval, maxPacketsPerInterval, _myServer->getPacketsPerClientPerInterval(),
nodeData->getMaxOctreePacketsPerSecond(), clientMaxPacketsPerInterval);
}
int extraPackingAttempts = 0;
bool completedScene = false;
while (somethingToSend && packetsSentThisInterval < maxPacketsPerInterval) {
if (_myServer->wantsDebugSending() && _myServer->wantsVerboseDebug()) {
qDebug("truePacketsSent=%d packetsSentThisInterval=%d maxPacketsPerInterval=%d server PPI=%d nodePPS=%d nodePPI=%d",
truePacketsSent, packetsSentThisInterval, maxPacketsPerInterval, _myServer->getPacketsPerClientPerInterval(),
nodeData->getMaxOctreePacketsPerSecond(), clientMaxPacketsPerInterval);
}
bool lastNodeDidntFit = false; // assume each node fits
if (!nodeData->nodeBag.isEmpty()) {
OctreeElement* subTree = nodeData->nodeBag.extract();
bool wantOcclusionCulling = nodeData->getWantOcclusionCulling();
CoverageMap* coverageMap = wantOcclusionCulling ? &nodeData->map : IGNORE_COVERAGE_MAP;
float voxelSizeScale = nodeData->getOctreeSizeScale();
int boundaryLevelAdjustClient = nodeData->getBoundaryLevelAdjust();
int boundaryLevelAdjust = boundaryLevelAdjustClient + (viewFrustumChanged && nodeData->getWantLowResMoving()
? LOW_RES_MOVING_ADJUST : NO_BOUNDARY_ADJUST);
bool isFullScene = ((!viewFrustumChanged || !nodeData->getWantDelta()) &&
nodeData->getViewFrustumJustStoppedChanging()) || nodeData->hasLodChanged();
EncodeBitstreamParams params(INT_MAX, &nodeData->getCurrentViewFrustum(), wantColor,
WANT_EXISTS_BITS, DONT_CHOP, wantDelta, lastViewFrustum,
wantOcclusionCulling, coverageMap, boundaryLevelAdjust, voxelSizeScale,
nodeData->getLastTimeBagEmpty(),
isFullScene, &nodeData->stats, _myServer->getJurisdiction());
_myServer->getOctree()->lockForRead();
nodeData->stats.encodeStarted();
bytesWritten = _myServer->getOctree()->encodeTreeBitstream(subTree, &_packetData, nodeData->nodeBag, params);
// If after calling encodeTreeBitstream() there are no nodes left to send, then we know we've
// sent the entire scene. We want to know this below so we'll actually write this content into
// the packet and send it
completedScene = nodeData->nodeBag.isEmpty();
// if we're trying to fill a full size packet, then we use this logic to determine if we have a DIDNT_FIT case.
if (_packetData.getTargetSize() == MAX_OCTREE_PACKET_DATA_SIZE) {
if (_packetData.hasContent() && bytesWritten == 0 &&
params.stopReason == EncodeBitstreamParams::DIDNT_FIT) {
lastNodeDidntFit = true;
}
} else {
// in compressed mode and we are trying to pack more... and we don't care if the _packetData has
// content or not... because in this case even if we were unable to pack any data, we want to drop
// below to our sendNow logic, but we do want to track that we attempted to pack extra
extraPackingAttempts++;
if (bytesWritten == 0 && params.stopReason == EncodeBitstreamParams::DIDNT_FIT) {
lastNodeDidntFit = true;
}
}
nodeData->stats.encodeStopped();
_myServer->getOctree()->unlock();
} else {
// If the bag was empty then we didn't even attempt to encode, and so we know the bytesWritten were 0
bytesWritten = 0;
somethingToSend = false; // this will cause us to drop out of the loop...
}
// If the last node didn't fit, but we're in compressed mode, then we actually want to see if we can fit a
// little bit more in this packet. To do this we write into the packet, but don't send it yet, we'll
// keep attempting to write in compressed mode to add more compressed segments
// We only consider sending anything if there is something in the _packetData to send... But
// if bytesWritten == 0 it means either the subTree couldn't fit or we had an empty bag... Both cases
// mean we should send the previous packet contents and reset it.
if (completedScene || lastNodeDidntFit) {
if (_packetData.hasContent()) {
// if for some reason the finalized size is greater than our available size, then probably the "compressed"
// form actually inflated beyond our padding, and in this case we will send the current packet, then
// write to out new packet...
int writtenSize = _packetData.getFinalizedSize()
+ (nodeData->getCurrentPacketIsCompressed() ? sizeof(OCTREE_PACKET_INTERNAL_SECTION_SIZE) : 0);
if (writtenSize > nodeData->getAvailable()) {
if (forceDebugging || (_myServer->wantsDebugSending() && _myServer->wantsVerboseDebug())) {
qDebug("about to call handlePacketSend() .... line: %d -- "
"writtenSize[%d] > available[%d] too big, sending packet as is.",
__LINE__, writtenSize, nodeData->getAvailable());
}
packetsSentThisInterval += handlePacketSend(node, nodeData, trueBytesSent, truePacketsSent);
}
if (forceDebugging || (_myServer->wantsDebugSending() && _myServer->wantsVerboseDebug())) {
qDebug(">>>>>> calling writeToPacket() available=%d compressedSize=%d uncompressedSize=%d target=%d",
nodeData->getAvailable(), _packetData.getFinalizedSize(),
_packetData.getUncompressedSize(), _packetData.getTargetSize());
}
nodeData->writeToPacket(_packetData.getFinalizedData(), _packetData.getFinalizedSize());
extraPackingAttempts = 0;
}
// If we're not running compressed, then we know we can just send now. Or if we're running compressed, but
// the packet doesn't have enough space to bother attempting to pack more...
bool sendNow = true;
if (nodeData->getCurrentPacketIsCompressed() &&
nodeData->getAvailable() >= MINIMUM_ATTEMPT_MORE_PACKING &&
extraPackingAttempts <= REASONABLE_NUMBER_OF_PACKING_ATTEMPTS) {
sendNow = false; // try to pack more
}
int targetSize = MAX_OCTREE_PACKET_DATA_SIZE;
if (sendNow) {
if (forceDebugging) {
qDebug("about to call handlePacketSend() .... line: %d -- sendNow = TRUE", __LINE__);
}
packetsSentThisInterval += handlePacketSend(node, nodeData, trueBytesSent, truePacketsSent);
if (wantCompression) {
targetSize = nodeData->getAvailable() - sizeof(OCTREE_PACKET_INTERNAL_SECTION_SIZE);
}
} else {
// If we're in compressed mode, then we want to see if we have room for more in this wire packet.
// but we've finalized the _packetData, so we want to start a new section, we will do that by
// resetting the packet settings with the max uncompressed size of our current available space
// in the wire packet. We also include room for our section header, and a little bit of padding
// to account for the fact that whenc compressing small amounts of data, we sometimes end up with
// a larger compressed size then uncompressed size
targetSize = nodeData->getAvailable() - sizeof(OCTREE_PACKET_INTERNAL_SECTION_SIZE) - COMPRESS_PADDING;
}
if (_myServer->wantsDebugSending() && _myServer->wantsVerboseDebug()) {
qDebug("line:%d _packetData.changeSettings() wantCompression=%s targetSize=%d",__LINE__,
debug::valueOf(nodeData->getWantCompression()), targetSize);
}
_packetData.changeSettings(nodeData->getWantCompression(), targetSize); // will do reset
}
}
// Here's where we can/should allow the server to send other data...
// send the environment packet
// TODO: should we turn this into a while loop to better handle sending multiple special packets
if (_myServer->hasSpecialPacketToSend(node)) {
trueBytesSent += _myServer->sendSpecialPacket(node);
truePacketsSent++;
packetsSentThisInterval++;
}
quint64 end = usecTimestampNow();
int elapsedmsec = (end - start)/1000;
quint64 endCompressCalls = OctreePacketData::getCompressContentCalls();
int elapsedCompressCalls = endCompressCalls - startCompressCalls;
quint64 endCompressTimeMsecs = OctreePacketData::getCompressContentTime() / 1000;
int elapsedCompressTimeMsecs = endCompressTimeMsecs - startCompressTimeMsecs;
if (elapsedmsec > 100) {
if (elapsedmsec > 1000) {
int elapsedsec = (end - start)/1000000;
qDebug("WARNING! packetLoop() took %d seconds [%d milliseconds %d calls in compress] "
"to generate %d bytes in %d packets %d nodes still to send",
elapsedsec, elapsedCompressTimeMsecs, elapsedCompressCalls,
trueBytesSent, truePacketsSent, nodeData->nodeBag.count());
} else {
qDebug("WARNING! packetLoop() took %d milliseconds [%d milliseconds %d calls in compress] "
"to generate %d bytes in %d packets, %d nodes still to send",
elapsedmsec, elapsedCompressTimeMsecs, elapsedCompressCalls,
trueBytesSent, truePacketsSent, nodeData->nodeBag.count());
}
} else if (_myServer->wantsDebugSending() && _myServer->wantsVerboseDebug()) {
qDebug("packetLoop() took %d milliseconds [%d milliseconds %d calls in compress] "
"to generate %d bytes in %d packets, %d nodes still to send",
elapsedmsec, elapsedCompressTimeMsecs, elapsedCompressCalls,
trueBytesSent, truePacketsSent, nodeData->nodeBag.count());
}
// if after sending packets we've emptied our bag, then we want to remember that we've sent all
// the voxels from the current view frustum
if (nodeData->nodeBag.isEmpty()) {
nodeData->updateLastKnownViewFrustum();
nodeData->setViewSent(true);
if (_myServer->wantsDebugSending() && _myServer->wantsVerboseDebug()) {
nodeData->map.printStats();
}
nodeData->map.erase(); // It would be nice if we could save this, and only reset it when the view frustum changes
}
if (_myServer->wantsDebugSending() && _myServer->wantsVerboseDebug()) {
qDebug("truePacketsSent=%d packetsSentThisInterval=%d maxPacketsPerInterval=%d "
"server PPI=%d nodePPS=%d nodePPI=%d",
truePacketsSent, packetsSentThisInterval, maxPacketsPerInterval,
_myServer->getPacketsPerClientPerInterval(), nodeData->getMaxOctreePacketsPerSecond(),
clientMaxPacketsPerInterval);
}
} // end if bag wasn't empty, and so we sent stuff...
return truePacketsSent;
}
