bool LocalNode::_cmdAddListener( Command& command )
{
NodeAddListenerPacket* packet =
command.getModifiable< NodeAddListenerPacket >();
ConnectionDescriptionPtr description =
new ConnectionDescription( packet->connectionData );
command.getNode()->addConnectionDescription( description );
if( command.getNode() != this )
return true;
EQASSERT( packet->connection );
ConnectionPtr connection = packet->connection;
packet->connection = 0;
connection->unref( CO_REFERENCED_PARAM );
_connectionNodes[ connection ] = this;
_incoming.addConnection( connection );
if( connection->getDescription()->type >= CONNECTIONTYPE_MULTICAST )
{
MCData data;
data.connection = connection;
data.node = this;
base::ScopedMutex<> mutex( _outMulticast );
_multicasts.push_back( data );
}
connection->acceptNB();
return true;
}
ConnectionPtr Connection::create( ConnectionDescriptionPtr description )
{
ConnectionPtr connection;
switch( description->type )
{
case CONNECTIONTYPE_TCPIP:
case CONNECTIONTYPE_SDP:
connection = new SocketConnection( description->type );
break;
case CONNECTIONTYPE_PIPE:
connection = new PipeConnection;
break;
#ifdef _WIN32
case CONNECTIONTYPE_NAMEDPIPE:
connection = new NamedPipeConnection;
break;
#endif
#ifdef EQ_INFINIBAND
case CONNECTIONTYPE_IB:
connection = new IBConnection;
break;
#endif
#ifdef EQ_PGM
case CONNECTIONTYPE_PGM:
connection = new PGMConnection;
break;
#endif
case CONNECTIONTYPE_RSP:
connection = new RSPConnection;
break;
#ifdef CO_USE_OFED
case CONNECTIONTYPE_RDMA:
connection = new RDMAConnection;
break;
#endif
#ifdef CO_USE_UDT
case CONNECTIONTYPE_UDT:
connection = new UDTConnection;
break;
#endif
default:
LBWARN << "Connection type " << description->type
<< " not supported" << std::endl;
return 0;
}
if( description->bandwidth == 0 )
description->bandwidth = connection->getDescription()->bandwidth;
connection->setDescription( description );
return connection;
}
void LocalNode::addListener( ConnectionPtr connection )
{
EQASSERT( isListening( ));
EQASSERT( connection->isListening( ));
connection->ref( CO_REFERENCED_PARAM );
NodeAddListenerPacket packet( connection );
Nodes nodes;
getNodes( nodes );
for( Nodes::iterator i = nodes.begin(); i != nodes.end(); ++i )
{
(*i)->send( packet, connection->getDescription()->toString( ));
}
}
uint32_t LocalNode::removeListenerNB( ConnectionPtr connection )
{
EQASSERT( isListening( ));
EQASSERT( connection->isListening( ));
connection->ref( CO_REFERENCED_PARAM );
NodeRemoveListenerPacket packet( connection, registerRequest( ));
Nodes nodes;
getNodes( nodes );
for( Nodes::iterator i = nodes.begin(); i != nodes.end(); ++i )
{
(*i)->send( packet, connection->getDescription()->toString( ));
}
return packet.requestID;
}
void Node::_removeMulticast( ConnectionPtr connection )
{
LBASSERT( connection->getDescription()->type >= CONNECTIONTYPE_MULTICAST );
lunchbox::ScopedMutex<> mutex( _impl->outMulticast );
if( _impl->outMulticast == connection )
_impl->outMulticast.data = 0;
else
{
for( MCDatas::iterator j = _impl->multicasts.begin();
j != _impl->multicasts.end(); ++j )
{
if( (*j).connection != connection )
continue;
_impl->multicasts.erase( j );
return;
}
}
}
bool LocalNode::_cmdRemoveListener( Command& command )
{
NodeRemoveListenerPacket* packet =
command.getModifiable< NodeRemoveListenerPacket >();
ConnectionDescriptionPtr description =
new ConnectionDescription( packet->connectionData );
EQCHECK( command.getNode()->removeConnectionDescription( description ));
if( command.getNode() != this )
return true;
EQASSERT( packet->connection );
ConnectionPtr connection = packet->connection;
packet->connection = 0;
connection->unref( CO_REFERENCED_PARAM );
if( connection->getDescription()->type >= CONNECTIONTYPE_MULTICAST )
{
base::ScopedMutex<> mutex( _outMulticast );
for( MCDatas::iterator i = _multicasts.begin();
i != _multicasts.end(); ++i )
{
if( i->connection == connection )
{
_multicasts.erase( i );
break;
}
}
}
_incoming.removeConnection( connection );
EQASSERT( _connectionNodes.find( connection ) != _connectionNodes.end( ));
_connectionNodes.erase( connection );
serveRequest( packet->requestID );
return true;
}
bool LocalNode::_handleData()
{
ConnectionPtr connection = _incoming.getConnection();
EQASSERT( connection.isValid( ));
NodePtr node;
ConnectionNodeHash::const_iterator i = _connectionNodes.find( connection );
if( i != _connectionNodes.end( ))
node = i->second;
EQASSERTINFO( !node || // unconnected node
*(node->_outgoing) == *connection || // correct UC connection
connection->getDescription()->type>=CONNECTIONTYPE_MULTICAST,
base::className( node ));
EQVERB << "Handle data from " << node << std::endl;
void* sizePtr( 0 );
uint64_t bytes( 0 );
const bool gotSize = connection->recvSync( &sizePtr, &bytes, false );
if( !gotSize ) // Some systems signal data on dead connections.
{
connection->recvNB( sizePtr, sizeof( uint64_t ));
return false;
}
EQASSERT( sizePtr );
const uint64_t size = *reinterpret_cast< uint64_t* >( sizePtr );
if( bytes == 0 ) // fluke signal
{
EQWARN << "Erronous network event on " << connection->getDescription()
<< std::endl;
_incoming.setDirty();
return false;
}
EQASSERT( size );
EQASSERTINFO( bytes == sizeof( uint64_t ), bytes );
EQASSERT( size > sizeof( size ));
if( node )
node->_lastReceive = getTime64();
Command& command = _commandCache.alloc( node, this, size );
uint8_t* ptr = reinterpret_cast< uint8_t* >(
command.getModifiable< Packet >()) + sizeof( uint64_t );
connection->recvNB( ptr, size - sizeof( uint64_t ));
const bool gotData = connection->recvSync( 0, 0 );
EQASSERT( gotData );
EQASSERT( command.isValid( ));
EQASSERT( command.isFree( ));
// start next receive
connection->recvNB( sizePtr, sizeof( uint64_t ));
if( !gotData )
{
EQERROR << "Incomplete packet read: " << command << std::endl;
return false;
}
// This is one of the initial packets during the connection handshake, at
// this point the remote node is not yet available.
EQASSERTINFO( node.isValid() ||
( command->type == PACKETTYPE_CO_NODE &&
( command->command == CMD_NODE_CONNECT ||
command->command == CMD_NODE_CONNECT_REPLY ||
command->command == CMD_NODE_ID )),
command << " connection " << connection );
_dispatchCommand( command );
return true;
}
void LocalNode::_handleDisconnect()
{
while( _handleData( )) ; // read remaining data off connection
ConnectionPtr connection = _incoming.getConnection();
ConnectionNodeHash::iterator i = _connectionNodes.find( connection );
if( i != _connectionNodes.end( ))
{
NodePtr node = i->second;
Command& command = _commandCache.alloc( node, this,
sizeof( NodeRemoveNodePacket ));
NodeRemoveNodePacket* packet =
command.getModifiable< NodeRemoveNodePacket >();
*packet = NodeRemoveNodePacket();
packet->node = node.get();
_dispatchCommand( command );
if( node->_outgoing == connection )
{
_objectStore->removeInstanceData( node->_id );
_connectionNodes.erase( i );
node->_state = STATE_CLOSED;
node->_outgoing = 0;
if( node->_outMulticast.data.isValid( ) )
_removeConnection( node->_outMulticast.data );
node->_outMulticast = 0;
node->_multicasts.clear();
EQINFO << node << " disconnected from " << *this << std::endl;
base::ScopedMutex< base::SpinLock > mutex( _nodes );
_nodes->erase( node->_id );
}
else
{
EQASSERT( connection->getDescription()->type >=
CONNECTIONTYPE_MULTICAST );
base::ScopedMutex<> mutex( _outMulticast );
if( node->_outMulticast == connection )
node->_outMulticast = 0;
else
{
for( MCDatas::iterator j = node->_multicasts.begin();
j != node->_multicasts.end(); ++j )
{
if( (*j).connection != connection )
continue;
node->_multicasts.erase( j );
break;
}
}
}
}
_removeConnection( connection );
EQINFO << "connection used " << connection->getRefCount() << std::endl;
}
bool LocalNode::_cmdID( Command& command )
{
EQASSERT( _inReceiverThread( ));
const NodeIDPacket* packet = command.get< NodeIDPacket >();
NodeID nodeID = packet->id;
if( command.getNode().isValid( ))
{
EQASSERT( nodeID == command.getNode()->getNodeID( ));
EQASSERT( command.getNode()->_outMulticast->isValid( ));
return true;
}
EQINFO << "handle ID " << packet << " node " << nodeID << std::endl;
ConnectionPtr connection = _incoming.getConnection();
EQASSERT( connection->getDescription()->type >= CONNECTIONTYPE_MULTICAST );
EQASSERT( _connectionNodes.find( connection ) == _connectionNodes.end( ));
NodePtr node;
if( nodeID == _id ) // 'self' multicast connection
node = this;
else
{
// No locking needed, only recv thread writes
NodeHash::const_iterator i = _nodes->find( nodeID );
if( i == _nodes->end( ))
{
// unknown node: create and add unconnected node
node = createNode( packet->nodeType );
std::string data = packet->data;
if( !node->deserialize( data ))
EQWARN << "Error during node initialization" << std::endl;
EQASSERTINFO( data.empty(), data );
{
base::ScopedMutex< base::SpinLock > mutex( _nodes );
_nodes.data[ nodeID ] = node;
}
EQVERB << "Added node " << nodeID << " with multicast "
<< connection << std::endl;
}
else
node = i->second;
}
EQASSERT( node.isValid( ));
EQASSERTINFO( node->_id == nodeID, node->_id << "!=" << nodeID );
base::ScopedMutex<> mutex( _outMulticast );
MCDatas::iterator i = node->_multicasts.begin();
for( ; i != node->_multicasts.end(); ++i )
{
if( (*i).connection == connection )
break;
}
if( node->_outMulticast->isValid( ))
{
if( node->_outMulticast.data == connection ) // connection already used
{
// nop
EQASSERT( i == node->_multicasts.end( ));
}
else // another connection is used as multicast connection, save this
{
if( i == node->_multicasts.end( ))
{
EQASSERT( _state == STATE_LISTENING );
MCData data;
data.connection = connection;
data.node = this;
node->_multicasts.push_back( data );
}
// else nop, already know connection
}
}
else
{
node->_outMulticast.data = connection;
if( i != node->_multicasts.end( ))
node->_multicasts.erase( i );
}
_connectionNodes[ connection ] = node;
EQINFO << "Added multicast connection " << connection << " from " << nodeID
<< " to " << _id << std::endl;
return true;
}