void context_base::wait( scheduler_i * sched )
{
CNC_ASSERT( !distributed() || distributor::distributed_env() || distributor::myPid() == 0 );
if( sched == NULL ) sched = m_scheduler;
sched->wait_loop();
if( subscribed() && sched->subscribed() ) {
if( distributor::myPid() == 0 ) cleanup_distributables( true );
sched->wait_loop();
}
}
bool GenericCommunicator::bcast_msg( serializer * ser, const int * rcvers, int nrecvrs )
{
VT_FUNC_I( "Dist::GenComm::bcast_msg" );
// Prepare the serializer for sending (add Size, CRC etc.)
CNC_ASSERT( ser && ser->is_packing() );
//BufferAccess::finalizePack( *ser );
// Bcast data:
return m_sendThread->pushForBcast( ser, rcvers, nrecvrs, m_globalIdShift );
}
void GenericCommunicator::bcast_msg( serializer * ser )
{
VT_FUNC_I( "Dist::GenComm::bcast_msg" );
// Prepare the serializer for sending (add Size, CRC etc.)
CNC_ASSERT( ser && ser->is_packing() );
//BufferAccess::finalizePack( *ser );
// Bcast data:
m_sendThread->pushForBcast( ser );
}
void SocketClientInitializer::init_itac_comm()
{
#ifdef CNC_WITH_ITAC
VT_FUNC_I( "Dist::Socket::init_itac_comm" );
UINT32 nBytes;
PAL_SockRes_t ret;
// Receive client ranks (local and global):
int itacRank[2];
ret = PAL_Recv( HERE, m_channel.m_socketCommData[0].m_recvSocket, itacRank, 2 * sizeof( int ), &nBytes, -1, false );
CNC_ASSERT( ret == PAL_SOCK_OK && nBytes == sizeof( itacRank ) );
int localRank = itacRank[0];
int globalRank = itacRank[1];
// Define process name in ITAC tracefile:
char clientName[128];
sprintf( clientName, "sClient%d", localRank );
// Determine VTcs contact string:
const char* vtContact;
VT_CLIENT_INIT( globalRank, NULL, &vtContact );
// Send VTcs contact string to host:
int length = (int)strlen( vtContact ) + 1;
ret = PAL_Send( HERE, m_channel.m_socketCommData[0].m_sendSocket, &length, sizeof( int ), &nBytes, -1 );
CNC_ASSERT( ret == PAL_SOCK_OK && nBytes == sizeof( length ) );
ret = PAL_Send( HERE, m_channel.m_socketCommData[0].m_sendSocket, vtContact, length, &nBytes, -1 );
CNC_ASSERT( ret == PAL_SOCK_OK && nBytes == length );
// Construct ITAC thread name:
char thrName[64];
sprintf( thrName, "RECV%d", m_channel.localId() );
// Initialize VTcs:
VT_INIT();
VT_THREAD_NAME( thrName );
#endif // CNC_WITH_ITAC
}
void GenericCommunicator::SendThread::sendTerminationRequest( int rcverLocalId, volatile bool * indicator )
{
CNC_ASSERT( 0 <= rcverLocalId && rcverLocalId < m_channel.numProcs() );
CNC_ASSERT( m_channel.isActive( rcverLocalId ) );
// Dummy serializer:
serializer ser( m_useCRC, true );
ser.set_mode_pack();
// Prepare it for sending: set its header to NULL
// which indicates the termination request.
BufferAccess::finalizePack( ser );
CNC_ASSERT( ser.get_body_size() == 0 );
// Send only the zero'd header:
m_channel.sendBytes( ser.get_header(), ser.get_header_size(), 0 /* body size */ , rcverLocalId );
// Indicate that the message has been sent successfully:
if ( indicator ) {
*indicator = true;
}
}
void GenericCommunicator::SendThread::pushForSend( serializer * ser, int rcverLocalId )
{
CNC_ASSERT( ser && rcverLocalId >= 0 );
#ifdef WITHOUT_SENDER_THREAD
send( ser, rcverLocalId );
cleanupSerializer( ser );
#else
# ifdef PRE_SEND_MSGS
rcverLocalId = send( ser, rcverLocalId );
# endif
m_sendQueue.push( SendItem( ser, rcverLocalId ) );
// nonzero ser and rcver id >= 0 indicate Send
#endif
}
void SocketClientInitializer::build_client_connections()
{
PAL_SockRes_t ret;
UINT32 nBytes;
int okay = 0;
do {
// Receive desired mode:
int mode;
ret = PAL_Recv( HERE, m_channel.m_socketCommData[0].m_recvSocket, &mode, sizeof( int ), &nBytes, -1, false );
CNC_ASSERT( ret == PAL_SOCK_OK && nBytes == sizeof( mode ) );
// Perform the mode's actions:
switch ( mode ) {
case SocketChannelInterface::SETUP_SEND_CONTACT_STRING: {
accept_client_connections();
break;
}
case SocketChannelInterface::SETUP_RECV_CONTACT_STRING: {
connect_to_other_client();
break;
}
case SocketChannelInterface::SETUP_DONE: {
okay = 1;
ret = PAL_Send( HERE, m_channel.m_socketCommData[0].m_sendSocket, &okay, sizeof( int ), &nBytes, -1 );
CNC_ASSERT( ret == PAL_SOCK_OK && nBytes == sizeof( okay ) );
break;
}
}
} while ( ! okay );
// Check whether all relevant sockets have been created:
for ( int j = 0; j < (int)m_channel.m_socketCommData.size(); ++j ) {
CNC_ASSERT( m_channel.m_socketCommData[j].m_sendSocket != PAL_INVALID_SOCKET || j == m_channel.localId() );
CNC_ASSERT( m_channel.m_socketCommData[j].m_recvSocket != PAL_INVALID_SOCKET || j == m_channel.localId() );
}
}
int MpiCommunicator::init( int minId, long thecomm_ )
{
VT_FUNC_I( "MpiCommunicator::init" );
assert( sizeof(thecomm_) >= sizeof(MPI_Comm) );
MPI_Comm thecomm = (MPI_Comm)thecomm_;
// turn wait mode on for intel mpi if possible
// this should greatly improve performance for intel mpi
PAL_SetEnvVar( "I_MPI_WAIT_MODE", "enable", 0);
int flag;
MPI_Initialized( &flag );
if ( ! flag ) {
int p;
//!! FIXME passing NULL ptr breaks mvapich1 mpi implementation
MPI_Init_thread( 0, NULL, MPI_THREAD_MULTIPLE, &p );
if( p != MPI_THREAD_MULTIPLE ) {
// can't use Speaker yet, need Channels to be inited
std::cerr << "[CnC] Warning: not MPI_THREAD_MULTIPLE (" << MPI_THREAD_MULTIPLE << "), but " << p << std::endl;
}
} else if( thecomm == 0 ) {
CNC_ABORT( "Process has already been initialized" );
}
MPI_Comm myComm = MPI_COMM_WORLD;
int rank;
MPI_Comm parentComm;
if( thecomm == 0 ) {
MPI_Comm_get_parent( &parentComm );
} else {
m_customComm = true;
m_exit0CallOk = false;
myComm = thecomm;
}
MPI_Comm_rank( myComm, &rank );
// father of all checks if he's requested to spawn processes:
if ( rank == 0 && parentComm == MPI_COMM_NULL ) {
// Ok, let's spawn the clients.
// I need some information for the startup.
// 1. Name of the executable (default is the current exe)
const char * _tmp = getenv( "CNC_MPI_SPAWN" );
if ( _tmp ) {
int nClientsToSpawn = atol( _tmp );
_tmp = getenv( "CNC_MPI_EXECUTABLE" );
std::string clientExe( _tmp ? _tmp : "" );
if( clientExe.empty() ) clientExe = PAL_GetProgname();
CNC_ASSERT( ! clientExe.empty() );
// 3. Special setting for MPI_Info: hosts
const char * clientHost = getenv( "CNC_MPI_HOSTS" );
// Prepare MPI_Info object:
MPI_Info clientInfo = MPI_INFO_NULL;
if ( clientHost ) {
MPI_Info_create( &clientInfo );
if ( clientHost ) {
MPI_Info_set( clientInfo, const_cast< char * >( "host" ), const_cast< char * >( clientHost ) );
// can't use Speaker yet, need Channels to be inited
std::cerr << "[CnC " << rank << "] Set MPI_Info_set( \"host\", \"" << clientHost << "\" )\n";
}
}
// Now spawn the client processes:
// can't use Speaker yet, need Channels to be inited
std::cerr << "[CnC " << rank << "] Spawning " << nClientsToSpawn << " MPI processes" << std::endl;
int* errCodes = new int[nClientsToSpawn];
MPI_Comm interComm;
int err = MPI_Comm_spawn( const_cast< char * >( clientExe.c_str() ),
MPI_ARGV_NULL, nClientsToSpawn,
clientInfo, 0, MPI_COMM_WORLD,
&interComm, errCodes );
delete [] errCodes;
if ( err ) {
// can't use Speaker yet, need Channels to be inited
std::cerr << "[CnC " << rank << "] Error in MPI_Comm_spawn. Skipping process spawning";
} else {
MPI_Intercomm_merge( interComm, 0, &myComm );
}
} // else {
// No process spawning
// MPI-1 situation: all clients to be started by mpiexec
// myComm = MPI_COMM_WORLD;
//}
}
if ( thecomm == 0 && parentComm != MPI_COMM_NULL ) {
// I am a child. Build intra-comm to the parent.
MPI_Intercomm_merge( parentComm, 1, &myComm );
}
MPI_Comm_rank( myComm, &rank );
CNC_ASSERT( m_channel == NULL );
MpiChannelInterface* myChannel = new MpiChannelInterface( use_crc(), myComm );
m_channel = myChannel;
int size;
MPI_Comm_size( myComm, &size );
// Are we on the host or on the remote side?
if ( rank == 0 ) {
if( size <= 1 ) {
Speaker oss( std::cerr ); oss << "Warning: no clients avabilable. Forgot to set CNC_MPI_SPAWN?";
}
// ==> HOST startup:
// This initializes the mpi environment in myChannel.
MpiHostInitializer hostInitializer( *myChannel );
hostInitializer.init_mpi_comm( myComm );
} else {
// ==> CLIENT startup:
// This initializes the mpi environment in myChannel.
MpiClientInitializer clientInitializer( *myChannel );
clientInitializer.init_mpi_comm( myComm );
}
{ Speaker oss( std::cerr ); oss << "MPI initialization complete (rank " << rank << ")."; }
// MPI_Barrier( myComm );
// Now the mpi specific setup is finished.
// Do the generic initialization stuff.
GenericCommunicator::init( minId );
return 0;
}
MpiCommunicator::~MpiCommunicator()
{
VT_FUNC_I( "MpiCommunicator::~MpiCommunicator" );
CNC_ASSERT( m_channel == 0 ); // was deleted in fini()
}
void MpiChannelInterface::recvBodyBytes( void * body, size_type bodySize, int senderLocalId )
{
VT_FUNC_I( "MPI::recvBodyBytes" );
CNC_ASSERT( 0 <= senderLocalId && senderLocalId < numProcs() );
MPI_Recv( body, bodySize, MPI_CHAR, senderLocalId, SECOND_MSG, m_communicator, MPI_STATUS_IGNORE );
}
void operator()( schedulable * stepInstance ) const
{
CNC_ASSERT( stepInstance );
stepInstance->scheduler().do_execute( stepInstance );
}
void SocketClientInitializer::init_socket_comm()
{
VT_FUNC_I( "Dist::Socket::init_socket_comm" );
UINT32 nBytes;
PAL_SockRes_t ret;
// Initialize socket layer (must be done only once ...)
PAL_SockInit( HERE );
// Create socket connection with the host.
// Resulting communicators are put into m_channel.m_socketCommData[0].
// Socket connections to the other clients will be set up
// later (in build_client_connections).
PAL_Socket_t newSocket;
// Predefined client id?
int myClientId = 0;
const char* cncSocketClientId = getenv( "CNC_SOCKET_CLIENT_ID" );
// envvar to be set from the client starter script,
// don't make it a config setting!
if ( cncSocketClientId ) {
myClientId = atoi( cncSocketClientId );
CNC_ASSERT( myClientId >= 1 );
}
// Create first socket connection to the host:
ret = PAL_Connect( HERE, m_hostContactStr, -1, &newSocket );
CNC_ASSERT( ret == PAL_SOCK_OK );
int clientIdArr[2];
clientIdArr[0] = ( ! cncSocketClientId ) ? 0 : 1; // defines 1st stage AND whether client id was set on the client
clientIdArr[1] = myClientId;
ret = PAL_Send( HERE, newSocket, clientIdArr, 2 * sizeof( int ), &nBytes, -1 );
CNC_ASSERT( ret == PAL_SOCK_OK && nBytes == sizeof( clientIdArr ) );
const int myClientIdOrig = myClientId;
int arr[2];
ret = PAL_Recv( HERE, newSocket, arr, 2 * sizeof( int ), &nBytes, -1, false );
CNC_ASSERT( ret == PAL_SOCK_OK && nBytes == sizeof( arr ) );
myClientId = arr[0];
int numProcsTotal = arr[1];
// Check client id:
CNC_ASSERT( ! cncSocketClientId || myClientId == myClientIdOrig );
CNC_ASSERT( myClientId >= 0 );
// ==> Now the number of relevant processes is known.
// Prepare the data structures accordingly !!!
m_channel.setLocalId( myClientId );
m_channel.setNumProcs( numProcsTotal );
m_channel.m_socketCommData[0].m_recvSocket = newSocket;
// Create second socket connection to the host:
ret = PAL_Connect( HERE, m_hostContactStr, -1, &m_channel.m_socketCommData[0].m_sendSocket );
CNC_ASSERT( ret == PAL_SOCK_OK );
clientIdArr[0] = 2; // defines 2nd stage
clientIdArr[1] = myClientId; // must now be >= 0
ret = PAL_Send( HERE, m_channel.m_socketCommData[0].m_sendSocket, clientIdArr, 2 * sizeof( int ), &nBytes, -1 );
CNC_ASSERT( ret == PAL_SOCK_OK && nBytes == sizeof( clientIdArr ) );
// Agree with the host on setup data like the number of worker threads.
exchange_setup_info();
// Setup connections between clients. Host will be coordinating this.
build_client_connections();
// Final step of initialization: setup of ITAC connections (VTcs):
init_itac_comm();
}
void SocketClientInitializer::connect_to_other_client()
{
PAL_SockRes_t ret;
UINT32 nBytes;
// Receive the contact string and the id of the client
// which I should connect to:
int arr[2];
ret = PAL_Recv( HERE, m_channel.m_socketCommData[0].m_recvSocket, arr, 2 * sizeof( int ), &nBytes, m_channel.m_timeout, false );
CNC_ASSERT( ret == PAL_SOCK_OK && nBytes == sizeof( arr ) );
int clientId = arr[0];
int contactStrLen = arr[1];
CNC_ASSERT( 0 < clientId && clientId < (int)m_channel.m_socketCommData.size() );
CNC_ASSERT( contactStrLen > 0 );
char* contactStr = new char[contactStrLen];
ret = PAL_Recv( HERE, m_channel.m_socketCommData[0].m_recvSocket, contactStr, contactStrLen, &nBytes, m_channel.m_timeout, false );
CNC_ASSERT( ret == PAL_SOCK_OK && nBytes == (UINT32)contactStrLen );
// Create first socket connection to the remote client:
CNC_ASSERT( m_channel.m_socketCommData[clientId].m_recvSocket == PAL_INVALID_SOCKET );
ret = PAL_Connect( HERE, contactStr, -1, &m_channel.m_socketCommData[clientId].m_recvSocket );
CNC_ASSERT( ret == PAL_SOCK_OK );
int myClientId = -m_channel.localId();
ret = PAL_Send( HERE, m_channel.m_socketCommData[clientId].m_recvSocket, &myClientId, sizeof( int ), &nBytes, -1 );
CNC_ASSERT( ret == PAL_SOCK_OK && nBytes == sizeof( myClientId ) );
ret = PAL_Recv( HERE, m_channel.m_socketCommData[clientId].m_recvSocket, &myClientId, sizeof( int ), &nBytes, -1, false );
CNC_ASSERT( ret == PAL_SOCK_OK && nBytes == sizeof( myClientId ) );
CNC_ASSERT( myClientId == m_channel.localId() );
// Create second socket connection to the host:
CNC_ASSERT( m_channel.m_socketCommData[clientId].m_sendSocket == PAL_INVALID_SOCKET );
ret = PAL_Connect( HERE, contactStr, -1, &m_channel.m_socketCommData[clientId].m_sendSocket );
CNC_ASSERT( ret == PAL_SOCK_OK );
ret = PAL_Send( HERE, m_channel.m_socketCommData[clientId].m_sendSocket, &myClientId, sizeof( int ), &nBytes, -1 );
CNC_ASSERT( ret == PAL_SOCK_OK && nBytes == sizeof( myClientId ) );
// Cleanup:
delete [] contactStr;
}
void SocketClientInitializer::accept_client_connections()
{
PAL_SockRes_t ret;
UINT32 nBytes;
int nClientsContacting;
ret = PAL_Recv( HERE, m_channel.m_socketCommData[0].m_recvSocket, &nClientsContacting, sizeof( int ), &nBytes, -1, false );
CNC_ASSERT( ret == PAL_SOCK_OK && nBytes == sizeof( nClientsContacting ) );
CNC_ASSERT( nClientsContacting > 0 );
int nConnectionsExpected = 2 * nClientsContacting;
// Create contact string:
std::string contactStr;
const char* CnCSocketHostname = Internal::Settings::get_string( "CNC_SOCKET_HOSTNAME", NULL );
UINT32 CnCSocketPort = Settings::get_int( "CNC_SOCKET_PORT", 0 );
PAL_Socket_t initialSocket;
ret = PAL_Listen( HERE, 0, nConnectionsExpected, CnCSocketHostname, CnCSocketPort, contactStr, &initialSocket );
CNC_ASSERT( ret == PAL_SOCK_OK );
// Send contact string back to host:
int contactStrLen = (int)contactStr.size() + 1;
ret = PAL_Send( HERE, m_channel.m_socketCommData[0].m_sendSocket, &contactStrLen, sizeof( int ), &nBytes, -1 );
CNC_ASSERT( ret == PAL_SOCK_OK && nBytes == sizeof( contactStrLen ) );
ret = PAL_Send( HERE, m_channel.m_socketCommData[0].m_sendSocket, contactStr.c_str(), contactStrLen, &nBytes, -1 );
CNC_ASSERT( ret == PAL_SOCK_OK && nBytes == (UINT32)contactStrLen );
//%%%%%%%%%%%%%%%%%%%%%%%%%%
// Wait for other clients to connect to this client.
// For each client, two connections are set up:
for ( int iConn = 0; iConn < nConnectionsExpected; ++iConn )
{
// Accept next connection.
// It may be the 1st or the 2nd connection to the respective client.
PAL_Socket_t newSocket;
int remoteClientId;
ret = PAL_Accept( HERE, initialSocket, -1, &newSocket );
CNC_ASSERT( ret == PAL_SOCK_OK );
ret = PAL_Recv( HERE, newSocket, &remoteClientId, sizeof( int ), &nBytes, m_channel.m_timeout, false );
CNC_ASSERT( ret == PAL_SOCK_OK && nBytes == sizeof( remoteClientId ) );
if ( remoteClientId < 0 ) {
// It is the first connection to this client.
// Send back the id to the client, and the total number
// of involved processes (host and clients):
remoteClientId = -remoteClientId;
CNC_ASSERT( remoteClientId < (int)m_channel.m_socketCommData.size() );
ret = PAL_Send( HERE, newSocket, &remoteClientId, sizeof( int ), &nBytes, -1 );
CNC_ASSERT( ret == PAL_SOCK_OK && nBytes == sizeof( remoteClientId ) );
CNC_ASSERT( m_channel.m_socketCommData[remoteClientId].m_sendSocket == PAL_INVALID_SOCKET );
m_channel.m_socketCommData[remoteClientId].m_sendSocket = newSocket;
} else {
// It is the second connection to this client.
CNC_ASSERT( remoteClientId < (int)m_channel.m_socketCommData.size() );
CNC_ASSERT( m_channel.m_socketCommData[remoteClientId].m_recvSocket == PAL_INVALID_SOCKET );
m_channel.m_socketCommData[remoteClientId].m_recvSocket = newSocket;
// #if 1
// // Print info:
// if ( ! restarting ) {
// if ( nRemainingClients > 0 ) {
// std::cerr << "--> established socket connection " << client + 1 << ", "
// << nRemainingClients << " still missing ...\n" << std::flush;
// } else {
// std::cerr << "--> established all socket connections.\n" << std::flush;
// }
// }
// #endif
}
}
// Cleanup:
SocketChannelInterface::closeSocket( initialSocket );
}
tbb::task * execute()
{
CNC_ASSERT( m_schedulable );
m_schedulable->scheduler().do_execute( m_schedulable );
return NULL;
}
void distributor::recv_msg( serializer *ser, int pid )
{
const int terminationId = -111111;
// Host initialization stuff:
if ( ! remote() && g_host_data == 0 ) {
CNC_ASSERT( ! g_host_communication_done );
g_host_data = new HostData( g_mySocketComm.numProcs() - 1 );
}
// Unpack message:
BufferAccess::initUnpack( *ser );
int dummy;
(*ser) & dummy;
printf( "PROC %d: received %d from %d\n", myPid(), dummy, pid );
fflush( stdout );
// Host checks for finalization messages:
if ( ! remote() && dummy == terminationId ) { // indicates that client is finished
CNC_ASSERT( 0 < pid && pid <= g_host_data->m_numClients );
CNC_ASSERT( g_host_data->m_clientsFinished[pid] == false );
g_host_data->m_clientsFinished[pid] = true;
++g_host_data->m_numClientsFinished;
if ( g_host_data->m_numClientsFinished == g_host_data->m_numClients ) {
g_host_communication_done = true;
// wakes up host application thread
}
}
// SENDS:
if ( g_recv_counter == 0 )
{
// Clients send some messages:
// [0 --> 1], then
// 1 --> 2, 2 --> 3, ..., N-1 --> N, N --> 0 [=host]
if ( remote() ) {
int dummy = ( myPid() + 1 ) * 123;
int recver = ( myPid() + 1 ) % numProcs();
serializer * ser = new_serializer( 0 );
(*ser) & dummy;
send_msg( ser, recver );
}
// last client makes a bcast:
if ( myPid() == numProcs() - 1 ) {
int dummy = 5555;
serializer * ser = new_serializer( 0 );
(*ser) & dummy;
#if 1
// bcast to all others:
bcast_msg( ser );
#else
// restricted bcast variant:
std::vector< int > rcverArr;
rcverArr.push_back( 0 );
bcast_msg( ser, rcverArr );
#endif
}
// Finally each client sends a termination message to the host:
if ( remote() ) {
int dummy = terminationId;
serializer * ser = new_serializer( 0 );
(*ser) & dummy;
send_msg( ser, 0 );
}
}
// Adjust recv counter:
++g_recv_counter;
}
