void QueueMaster::push( const QueueItemPacket& packet )
{
LBASSERT( packet.size >= sizeof( QueueItemPacket ));
LBASSERT( packet.command == CMD_QUEUE_ITEM );
Command& command = _impl->cache.alloc( getLocalNode(), getLocalNode(),
packet.size );
QueueItemPacket* queuePacket = command.getModifiable< QueueItemPacket >();
memcpy( queuePacket, &packet, packet.size );
queuePacket->objectID = getID();
command.retain();
_impl->queue.push( &command );
}
bool Config::init( const uint128_t& initID )
{
EQASSERT( !_running );
_currentFrame = 0;
_unlockedFrame = 0;
_finishedFrame = 0;
_frameTimes.clear();
co::LocalNodePtr localNode = getLocalNode();
ConfigInitPacket packet;
packet.requestID = localNode->registerRequest();
packet.initID = initID;
send( getServer(), packet );
ClientPtr client = getClient();
while( !localNode->isRequestServed( packet.requestID ))
client->processCommand();
localNode->waitRequest( packet.requestID, _running );
localNode->enableSendOnRegister();
if( _running )
handleEvents();
else
EQWARN << "Config initialization failed: " << getError() << std::endl
<< " Consult client log for further information" << std::endl;
return _running;
}
void TileQueue::cycleData( const uint32_t frameNumber, const Compound* compound)
{
for( unsigned i = 0; i < NUM_EYES; ++i )
{
if( !compound->isInheritActive( Eye( 1<<i )))// eye pass not used
{
_queueMaster[i] = 0;
continue;
}
// reuse unused queues
LatencyQueue* queue = _queues.empty() ? 0 : _queues.back();
const uint32_t latency = getAutoObsolete();
const uint32_t dataAge = queue ? queue->_frameNumber : 0;
if( queue && dataAge < frameNumber-latency && frameNumber > latency )
// not used anymore
_queues.pop_back();
else // still used - allocate new data
{
queue = new LatencyQueue;
getLocalNode()->registerObject( &queue->_queue );
queue->_queue.setAutoObsolete( 1 ); // current + in use by render nodes
}
queue->_queue.clear();
queue->_frameNumber = frameNumber;
_queues.push_front( queue );
_queueMaster[i] = queue;
}
}
void Config::deregisterObject( co::Object* object )
{
EQASSERT( object )
EQASSERT( object->isMaster( ));
if( !object->isAttached( )) // not registered
return;
const uint32_t latency = getLatency();
ClientPtr client = getClient();
if( latency == 0 || !_running || !object->isBuffered( )) // OPT
{
client->deregisterObject( object );
return;
}
// Keep a distributed object latency frames.
// Replaces the object with a dummy proxy object using the
// existing master change manager.
ConfigSwapObjectPacket packet;
packet.requestID = getLocalNode()->registerRequest();
packet.object = object;
send( client, packet );
client->waitRequest( packet.requestID );
}
bool Config::exit()
{
update();
finishAllFrames();
co::LocalNodePtr localNode = getLocalNode();
localNode->disableSendOnRegister();
ConfigExitPacket packet;
packet.requestID = localNode->registerRequest();
send( getServer(), packet );
ClientPtr client = getClient();
while( !localNode->isRequestServed( packet.requestID ))
client->processCommand();
bool ret = false;
localNode->waitRequest( packet.requestID, ret );
while( tryNextEvent( )) /* flush all pending events */ ;
if( _lastEvent )
_lastEvent->release();
_eventQueue.flush();
_lastEvent = 0;
_running = false;
return ret;
}
void Config::_stopNodes()
{
// wait for the nodes to stop, destroy entities, disconnect
Nodes stoppingNodes;
const Nodes& nodes = getNodes();
for( Nodes::const_iterator i = nodes.begin(); i != nodes.end(); ++i )
{
Node* node = *i;
const State state = node->getState();
if( state != STATE_STOPPED && state != STATE_FAILED )
continue;
LBASSERT( !node->isActive() || state == STATE_FAILED );
if( node->isApplicationNode( ))
continue;
co::NodePtr netNode = node->getNode();
if( !netNode ) // already disconnected
continue;
LBLOG( LOG_INIT ) << "Exiting node" << std::endl;
if( state == STATE_FAILED )
node->setState( STATE_STOPPED );
stoppingNodes.push_back( node );
LBASSERT( netNode.isValid( ));
netNode->send( fabric::CMD_SERVER_DESTROY_CONFIG )
<< getID() << LB_UNDEFINED_UINT32;
netNode->send( fabric::CMD_CLIENT_EXIT );
}
// now wait that the render clients disconnect
uint32_t nSleeps = 50; // max 5 seconds for all clients
for( Nodes::const_iterator i = stoppingNodes.begin();
i != stoppingNodes.end(); ++i )
{
Node* node = *i;
co::NodePtr netNode = node->getNode();
node->setNode( 0 );
if( nSleeps )
while( netNode->isConnected() && --nSleeps )
lunchbox::sleep( 100 ); // ms
if( netNode->isConnected( ))
{
co::LocalNodePtr localNode = getLocalNode();
LBASSERT( localNode.isValid( ));
LBWARN << "Forcefully disconnecting exited render client node"
<< std::endl;
localNode->disconnect( netNode );
}
LBLOG( LOG_INIT ) << "Disconnected node" << std::endl;
}
}
void Pipe::_stopTransferThread()
{
if( _impl->transferThread.isStopped( ))
return;
send( getLocalNode(), fabric::CMD_PIPE_EXIT_TRANSFER_THREAD );
_impl->transferThread.join();
}
void QueueMaster::attach( const UUID& id, const uint32_t instanceID )
{
Object::attach( id, instanceID );
CommandQueue* queue = getLocalNode()->getCommandThreadQueue();
registerCommand( CMD_QUEUE_GET_ITEM,
CommandFunc< detail::QueueMaster >(
_impl, &detail::QueueMaster::cmdGetItem ), queue );
}
void Config::notifyAttached()
{
fabric::Object::notifyAttached();
EQASSERT( !_appNode )
EQASSERT( getAppNodeID().isGenerated() )
co::LocalNodePtr localNode = getLocalNode();
_appNode = localNode->connect( getAppNodeID( ));
EQASSERTINFO( _appNode, "Connection to application node failed -- " <<
"misconfigured connections on appNode?" );
}
void QueueSlave::applyInstanceData( co::DataIStream& is )
{
NodeID masterNodeID;
is >> _impl->masterInstanceID >> masterNodeID;
LBASSERT( masterNodeID != 0 );
LBASSERT( !_impl->master );
LocalNodePtr localNode = getLocalNode();
_impl->master = localNode->connect( masterNodeID );
}
void QueueSlave::applyInstanceData( co::DataIStream& is )
{
uint128_t masterNodeID;
is >> _impl->masterInstanceID >> masterNodeID;
EQASSERT( masterNodeID != NodeID::ZERO );
EQASSERT( !_impl->master );
LocalNodePtr localNode = getLocalNode();
_impl->master = localNode->connect( masterNodeID );
}
bool Config::_cmdCreateReply( co::ICommand& cmd )
{
co::ObjectICommand command( cmd );
LB_TS_THREAD( _cmdThread );
LB_TS_NOT_THREAD( _mainThread );
getLocalNode()->serveRequest( command.read< uint32_t >( ));
return true;
}
bool Config::_cmdExitReply( co::Command& command )
{
const ConfigExitReplyPacket* packet =
command.get<ConfigExitReplyPacket>();
EQVERB << "handle exit reply " << packet << std::endl;
_exitMessagePump();
getLocalNode()->serveRequest( packet->requestID, (void*)(packet->result) );
return true;
}
bool Config::_cmdSwapObject( co::Command& command )
{
const ConfigSwapObjectPacket* packet =
command.get<ConfigSwapObjectPacket>();
EQVERB << "Cmd swap object " << packet << std::endl;
co::Object* object = packet->object;
LatencyObject* latencyObject = new LatencyObject( object->getChangeType(),
object->chooseCompressor(),
_currentFrame + getLatency() + 1 );
getLocalNode()->swapObject( object, latencyObject );
{
co::base::ScopedFastWrite mutex( _latencyObjects );
_latencyObjects->push_back( latencyObject );
}
EQASSERT( packet->requestID != EQ_UNDEFINED_UINT32 );
getLocalNode()->serveRequest( packet->requestID );
return true;
}
bool Config::_cmdInitReply( co::Command& command )
{
const ConfigInitReplyPacket* packet =
command.get<ConfigInitReplyPacket>();
EQVERB << "handle init reply " << packet << std::endl;
sync( packet->version );
getLocalNode()->serveRequest( packet->requestID, (void*)(packet->result) );
return true;
}
bool Config::_cmdCheckFrame( co::ICommand& cmd )
{
const int64_t lastInterval = getServer()->getTime() - _lastCheck;
_lastCheck = getServer()->getTime();
co::ObjectICommand command( cmd );
LBVERB << "Check nodes for frame finish " << command << std::endl;
const uint32_t frameNumber = command.read< uint32_t >();
const uint32_t timeout = getTimeout();
bool retry = false;
const Nodes& nodes = getNodes();
for( Nodes::const_iterator i = nodes.begin(); i != nodes.end(); ++i )
{
Node* node = *i;
if( node->isRunning() && node->isActive() )
{
co::NodePtr netNode = node->getNode();
if ( netNode->isClosed() )
continue;
if ( node->getFinishedFrame() >= frameNumber )
continue;
const int64_t interval = getServer()->getTime() -
netNode->getLastReceiveTime();
getLocalNode()->ping( netNode );
// TODO?: handle timed out nodes.
// currently we get a false positive due to lack of communication
// from client to server. we do not get ping responses in time.
// running clients should inform the server about their status with
// a timeout/2 period.
if ( interval > timeout && lastInterval <= timeout )
continue;
// retry
LBINFO << "Retry waiting for node " << node->getName()
<< " to finish frame " << frameNumber << " last seen "
<< interval << " ms ago" << " last run " << lastInterval
<< std::endl;
retry = true;
// else node timeout
}
}
if( retry )
return true;
send( command.getRemoteNode(), fabric::CMD_CONFIG_FRAME_FINISH )
<< _currentFrame;
return true;
}
void Pipe::cancelThread()
{
_stopTransferThread();
if( !_impl->thread )
return;
// local command dispatching
co::ObjectOCommand( this, getLocalNode(), fabric::CMD_PIPE_EXIT_THREAD,
co::COMMANDTYPE_OBJECT, getID(), EQ_INSTANCE_ALL );
}
void TileQueue::flush()
{
unsetData();
while (!_queues.empty())
{
LatencyQueue* queue = _queues.front();
_queues.pop_front();
getLocalNode()->deregisterObject(&queue->_queue);
delete queue;
}
}
void Frame::flush()
{
unsetData();
while( !_datas.empty( ))
{
FrameData* data = _datas.front();
_datas.pop_front();
getLocalNode()->deregisterObject( data );
delete data;
}
}
uint32_t Config::finishFrame()
{
ClientPtr client = getClient();
const uint32_t latency = getLatency();
const uint32_t frameToFinish = (_currentFrame >= latency) ?
_currentFrame - latency : 0;
ConfigStatistics stat( Statistic::CONFIG_FINISH_FRAME, this );
stat.event.data.statistic.frameNumber = frameToFinish;
{
ConfigStatistics waitStat( Statistic::CONFIG_WAIT_FINISH_FRAME, this );
waitStat.event.data.statistic.frameNumber = frameToFinish;
// local draw sync
if( _needsLocalSync( ))
while( _unlockedFrame < _currentFrame )
client->processCommand();
// local node finish (frame-latency) sync
const Nodes& nodes = getNodes();
if( !nodes.empty( ))
{
EQASSERT( nodes.size() == 1 );
const Node* node = nodes.front();
while( node->getFinishedFrame() < frameToFinish )
client->processCommand();
}
// global sync
const uint32_t timeout = getTimeout();
co::base::Clock time;
const int64_t pingTimeout = co::Global::getKeepaliveTimeout();
while( !_finishedFrame.timedWaitGE( frameToFinish, pingTimeout ))
{
if( time.getTime64() >= timeout || !getLocalNode()->pingIdleNodes())
{
EQWARN << "Timeout waiting for nodes to finish frame "
<< frameToFinish << std::endl;
break;
}
}
}
handleEvents();
_updateStatistics( frameToFinish );
_releaseObjects();
EQLOG( co::base::LOG_ANY ) << "---- Finished Frame --- " << frameToFinish
<< " (" << _currentFrame << ')' << std::endl;
return frameToFinish;
}
void Pipe::exitThread()
{
_stopTransferThread();
if( !_impl->thread )
return;
send( getLocalNode(), fabric::CMD_PIPE_EXIT_THREAD );
_impl->thread->join();
delete _impl->thread;
_impl->thread = 0;
}
void View::detach()
{
// if pipe is not running, detach comes from _flushViews in state stopping
// Don't send command to stopping pipe (see issue #11)
if( _pipe && _pipe->isRunning( ))
{
// local command dispatching
co::ObjectOCommand( _pipe, getLocalNode(), fabric::CMD_PIPE_DETACH_VIEW,
co::COMMANDTYPE_OBJECT, _pipe->getID(),
CO_INSTANCE_ALL ) << getID();
}
Super::detach();
}
lunchbox::Request< void > Config::_createConfig( Node* node )
{
LBASSERT( !node->isApplicationNode( ));
LBASSERT( node->isActive( ));
// create config on each non-app node
// app-node already has config from chooseConfig
lunchbox::Request< void > request = getLocalNode()->registerRequest<void>();
node->getNode()->send( fabric::CMD_SERVER_CREATE_CONFIG )
<< co::ObjectVersion( this ) << request;
return request;
}
void View::detach()
{
if( _pipe )
{
co::LocalNodePtr localNode = getLocalNode();
co::Command& command =
localNode->allocCommand( sizeof( PipeDetachViewPacket ));
PipeDetachViewPacket* packet = command.get< PipeDetachViewPacket >();
*packet = PipeDetachViewPacket( getID( ));
_pipe->dispatchCommand( command );
}
Super::detach();
}
bool Config::_cmdUpdate( co::ICommand& cmd )
{
co::ObjectICommand command( cmd );
LBVERB << "handle config update " << command << std::endl;
const uint32_t versionID = command.read< uint32_t >();
const uint32_t finishID = command.read< uint32_t >();
sync();
commit();
co::NodePtr node = command.getRemoteNode();
if( !_needsFinish )
{
send( node, fabric::CMD_CONFIG_UPDATE_VERSION )
<< getVersion() << versionID << finishID << LB_UNDEFINED_UINT32;
return true;
}
co::LocalNodePtr localNode = getLocalNode();
lunchbox::Request< void > request = localNode->registerRequest< void >();
send( node, fabric::CMD_CONFIG_UPDATE_VERSION )
<< getVersion() << versionID << finishID << request;
_flushAllFrames();
_finishedFrame.waitEQ( _currentFrame ); // wait for render clients idle
request.wait(); // wait for app sync
_needsFinish = false;
const bool canFail = (getIAttribute( IATTR_ROBUSTNESS ) != OFF);
const bool result = _updateRunning( canFail );
if( !result && !canFail )
{
LBWARN << "Config update failed, exiting config" << std::endl;
exit();
}
const uint128_t version = commit();
send( command.getRemoteNode(), fabric::CMD_CONFIG_UPDATE_REPLY )
<< version << command.read< uint32_t >() << result;
return true;
}
bool Pipe::_cmdConfigExit( co::ICommand& cmd )
{
co::ObjectICommand command( cmd );
LB_TS_THREAD( _pipeThread );
LBLOG( LOG_INIT ) << "TASK pipe config exit " << command << std::endl;
_impl->state = STATE_STOPPING; // needed in View::detach (from _flushViews)
// send before node gets a chance to send its destroy command
getNode()->send( getLocalNode(), fabric::CMD_NODE_DESTROY_PIPE ) << getID();
// Flush views before exit since they are created after init
// - application may need initialized pipe to exit
// - configExit can't access views since all channels are gone already
_flushViews();
_flushQueues();
_impl->state = configExit() ? STATE_STOPPED : STATE_FAILED;
return true;
}
void Frame::cycleData(const uint32_t frameNumber, const Compound* compound)
{
_masterFrameData = 0;
for (unsigned i = 0; i < NUM_EYES; ++i)
{
_inputFrames[i].clear();
const Eye eye = Eye(1 << i);
if (!compound->isInheritActive(eye)) // eye pass not used
{
_frameData[i] = 0;
continue;
}
// reuse unused frame data
FrameData* data = _datas.empty() ? 0 : _datas.back();
const uint32_t latency = getAutoObsolete();
const uint32_t dataAge = data ? data->getFrameNumber() : 0;
if (data && dataAge < frameNumber - latency && frameNumber > latency)
// not used anymore
_datas.pop_back();
else // still used - allocate new data
{
data = new FrameData;
getLocalNode()->registerObject(data);
data->setAutoObsolete(1); // current + in use by render nodes
}
data->setFrameNumber(frameNumber);
_datas.push_front(data);
_frameData[i] = data;
_getInputNodes(i).clear();
_getInputNetNodes(i).clear();
if (!_masterFrameData)
_masterFrameData = data;
}
}
bool Window::_cmdConfigExit( co::ICommand& cmd )
{
co::ObjectICommand command( cmd );
LBLOG( LOG_INIT ) << "TASK window config exit " << command << std::endl;
if( _state != STATE_STOPPED )
{
if( getPipe()->isRunning( ) && _systemWindow )
{
makeCurrent();
getPipe()->flushFrames( _objectManager );
}
// else emergency exit, no context available.
_state = configExit() ? STATE_STOPPED : STATE_FAILED;
}
getPipe()->send( getLocalNode(),
fabric::CMD_PIPE_DESTROY_WINDOW ) << getID();
return true;
}
bool Node::_cmdConfigExit( co::Command& command )
{
LB_TS_THREAD( _nodeThread );
LBLOG( LOG_INIT ) << "Node exit "
<< command.get<NodeConfigExitPacket>() << std::endl;
const Pipes& pipes = getPipes();
for( Pipes::const_iterator i = pipes.begin(); i != pipes.end(); ++i )
{
Pipe* pipe = *i;
pipe->waitExited();
}
_state = configExit() ? STATE_STOPPED : STATE_FAILED;
transmitter.getQueue().wakeup();
transmitter.join();
_flushObjects();
ConfigDestroyNodePacket destroyPacket( getID( ));
getConfig()->send( getLocalNode(), destroyPacket );
return true;
}
void Node::_setAffinity()
{
const int32_t affinity = getIAttribute( IATTR_HINT_AFFINITY );
switch( affinity )
{
case OFF:
break;
case AUTO:
// TODO
LBVERB << "No automatic thread placement for node threads "
<< std::endl;
break;
default:
co::LocalNodePtr node = getLocalNode();
send( node, fabric::CMD_NODE_SET_AFFINITY ) << affinity;
node->setAffinity( affinity );
break;
}
}
