int MpiChannelInterface::sendBytes( void * data, size_type headerSize, size_type bodySize, int rcverLocalId )
{
VT_FUNC_I( "MPI::sendBytes" );
CNC_ASSERT( 0 <= rcverLocalId && rcverLocalId < numProcs() );
if( bodySize + headerSize > INT_MAX ) {
std::cerr << "MPI_Get_count doesn't allow a count > " << INT_MAX << ". No workaround implemented yet." << std::endl;
MPI_Abort( m_communicator, 1234 );
}
char* header_data = static_cast<char*>( data );
MPI_Request request = 0;
if( headerSize+bodySize < BUFF_SIZE ) {
MPI_Isend( header_data, headerSize+bodySize, MPI_CHAR, rcverLocalId, FIRST_MSG, m_communicator, &request );
} else {
CNC_ASSERT( bodySize > 0 );
// header Tag should not be equal to localId()
MPI_Send( header_data, headerSize, MPI_CHAR, rcverLocalId, FIRST_MSG, m_communicator );
char * body_data = header_data+headerSize;
MPI_Isend( body_data, bodySize, MPI_CHAR, rcverLocalId, SECOND_MSG, m_communicator, &request );
}
// { Speaker oss; oss << "sendBytes " << headerSize << " " << bodySize; }
#ifdef PRE_SEND_MSGS
return request;
#else
MPI_Wait( &request, MPI_STATUS_IGNORE );
return 0;
#endif
}
serializer * MpiChannelInterface::waitForAnyClient( int & senderLocalId )
{
VT_FUNC_I( "MPI::waitForAnyClient" );
MPI_Status status;
MPI_Wait( &m_request, &status );
senderLocalId = status.MPI_SOURCE;
CNC_ASSERT( 0 <= senderLocalId && senderLocalId < numProcs() );
int _cnt;
MPI_Get_count( &status, MPI_CHAR, &_cnt );
size_type _bodySize = m_ser1->unpack_header(); // throws an exception in case of error
CNC_ASSERT( _bodySize + m_ser1->get_header_size() == _cnt || m_ser1->get_header_size() == _cnt );
// if we did not receive the body yet, we need to do so now
if( _bodySize != 0 ) {
CNC_ASSERT( _bodySize != Buffer::invalid_size );
BufferAccess::acquire( *m_ser1, _bodySize ); // this is needed even if all is received: sets current pointer in buffer
if( _cnt == m_ser1->get_header_size() ) {
// Enlarge the buffer if needed
MPI_Recv( m_ser1->get_body(), _bodySize, MPI_CHAR, senderLocalId, SECOND_MSG, m_communicator, MPI_STATUS_IGNORE );
}
}
std::swap( m_ser1, m_ser2 ); // double buffer exchange
m_ser1->set_mode_unpack();
MPI_Irecv( m_ser1->get_header(), BufferAccess::capacity( *m_ser1 ), MPI_CHAR, MPI_ANY_SOURCE, FIRST_MSG, m_communicator, &m_request );
// { Speaker oss; oss << "recvBytes " << _bodySize; }
return _bodySize != 0 ? m_ser2 : NULL;
}
void GenericCommunicator::fini()
{
VT_FUNC_I( "Dist::GenComm::fini" );
// Something to do at all?
if ( ! m_hasBeenInitialized ) {
return;
} else {
m_hasBeenInitialized = false;
}
// Host sends termination requests to the remote clients.
// Each client will send a response.
if( m_channel->localId() == 0 ) {
for ( int client = 1; client < numProcs(); ++client ) {
send_termination_request( client );
}
}
// Stop sender and receiver threads:
if ( m_recvThread ) {
m_recvThread->stop();
}
if ( m_sendThread ) {
m_sendThread->stop();
}
// Cleanup:
delete m_recvThread;
m_recvThread = NULL;
delete m_sendThread;
m_sendThread = NULL;
// Cleanup ITAC stuff:
VT_FINALIZE();
}
int compute_on( const int & p, my_context & ) const
{
return ( 1 + p / ( 1000000 / 20 ) ) % numProcs();//( p * 5 / 3 + 1 ) % 5;
}
void MpiChannelInterface::recvBodyBytes( void * body, size_type bodySize, int senderLocalId )
{
VT_FUNC_I( "MPI::recvBodyBytes" );
CNC_ASSERT( 0 <= senderLocalId && senderLocalId < numProcs() );
MPI_Recv( body, bodySize, MPI_CHAR, senderLocalId, SECOND_MSG, m_communicator, MPI_STATUS_IGNORE );
}
void distributor::recv_msg( serializer *ser, int pid )
{
const int terminationId = -111111;
// Host initialization stuff:
if ( ! remote() && g_host_data == 0 ) {
CNC_ASSERT( ! g_host_communication_done );
g_host_data = new HostData( g_mySocketComm.numProcs() - 1 );
}
// Unpack message:
BufferAccess::initUnpack( *ser );
int dummy;
(*ser) & dummy;
printf( "PROC %d: received %d from %d\n", myPid(), dummy, pid );
fflush( stdout );
// Host checks for finalization messages:
if ( ! remote() && dummy == terminationId ) { // indicates that client is finished
CNC_ASSERT( 0 < pid && pid <= g_host_data->m_numClients );
CNC_ASSERT( g_host_data->m_clientsFinished[pid] == false );
g_host_data->m_clientsFinished[pid] = true;
++g_host_data->m_numClientsFinished;
if ( g_host_data->m_numClientsFinished == g_host_data->m_numClients ) {
g_host_communication_done = true;
// wakes up host application thread
}
}
// SENDS:
if ( g_recv_counter == 0 )
{
// Clients send some messages:
// [0 --> 1], then
// 1 --> 2, 2 --> 3, ..., N-1 --> N, N --> 0 [=host]
if ( remote() ) {
int dummy = ( myPid() + 1 ) * 123;
int recver = ( myPid() + 1 ) % numProcs();
serializer * ser = new_serializer( 0 );
(*ser) & dummy;
send_msg( ser, recver );
}
// last client makes a bcast:
if ( myPid() == numProcs() - 1 ) {
int dummy = 5555;
serializer * ser = new_serializer( 0 );
(*ser) & dummy;
#if 1
// bcast to all others:
bcast_msg( ser );
#else
// restricted bcast variant:
std::vector< int > rcverArr;
rcverArr.push_back( 0 );
bcast_msg( ser, rcverArr );
#endif
}
// Finally each client sends a termination message to the host:
if ( remote() ) {
int dummy = terminationId;
serializer * ser = new_serializer( 0 );
(*ser) & dummy;
send_msg( ser, 0 );
}
}
// Adjust recv counter:
++g_recv_counter;
}
//------------------------------------------------------------------------
int mirrorCommPattern(MPI_Comm comm, comm_map* inPattern, comm_map*& outPattern)
{
#ifdef FEI_SER
(void)inPattern;
(void)outPattern;
#else
int localP = localProc(comm);
int numP = numProcs(comm);
if (numP < 2) return(0);
std::vector<int> buf(numP*2, 0);
int numInProcs = inPattern->getMap().size();
std::vector<int> inProcs(numInProcs);
fei::copyKeysToVector(inPattern->getMap(), inProcs);
std::vector<int> outProcs;
int err = mirrorProcs(comm, inProcs, outProcs);
if (err != 0) ERReturn(-1);
std::vector<int> recvbuf(outProcs.size(), 0);
outPattern = new comm_map(0,1);
MPI_Datatype mpi_ttype = fei::mpiTraits<int>::mpi_type();
//now recv a length (the contents of buf[i]) from each "out-proc", which
//will be the length of the equation data that will also be recvd from that
//proc.
std::vector<MPI_Request> mpiReqs(outProcs.size());
std::vector<MPI_Status> mpiStss(outProcs.size());
MPI_Request* requests = &mpiReqs[0];
MPI_Status* statuses = &mpiStss[0];
int firsttag = 11117;
int offset = 0;
int* outProcsPtr = &outProcs[0];
for(unsigned i=0; i<outProcs.size(); ++i) {
if (MPI_Irecv(&(recvbuf[i]), 1, MPI_INT, outProcsPtr[i], firsttag,
comm, &requests[offset++]) != MPI_SUCCESS) ERReturn(-1);
}
comm_map::map_type& in_row_map = inPattern->getMap();
comm_map::map_type::iterator
in_iter = in_row_map.begin(),
in_end = in_row_map.end();
int* inProcsPtr = &inProcs[0];
for(int ii=0; in_iter!= in_end; ++in_iter, ++ii) {
comm_map::row_type* in_row = in_iter->second;
buf[ii] = in_row->size();
if (MPI_Send(&(buf[ii]), 1, MPI_INT, inProcsPtr[ii], firsttag,
comm) != MPI_SUCCESS) ERReturn(-1);
}
int numOutProcs = outProcs.size();
MPI_Waitall(numOutProcs, requests, statuses);
std::vector<int> lengths(numOutProcs);
int totalRecvLen = 0;
offset = 0;
for(int ii=0; ii<numOutProcs; ++ii) {
if (recvbuf[ii] > 0) {
lengths[offset++] = recvbuf[ii];
totalRecvLen += recvbuf[ii];
}
}
//now we need to create the space into which we'll receive the
//lists that other procs send to us.
std::vector<int> recvData(totalRecvLen, 999999);
int tag2 = 11118;
offset = 0;
for(int ii=0; ii<numOutProcs; ++ii) {
CHK_MPI(MPI_Irecv(&(recvData[offset]), lengths[ii], mpi_ttype,
outProcs[ii], tag2, comm, &requests[ii]) );
offset += lengths[ii];
}
std::vector<int> sendList;
in_iter = in_row_map.begin();
for(int ii=0; in_iter != in_end; ++in_iter,++ii) {
if (inProcs[ii] == localP) {
continue;
}
sendList.resize(in_iter->second->size());
fei::copySetToArray(*(in_iter->second), sendList.size(), &sendList[0]);
CHK_MPI(MPI_Send(&sendList[0], sendList.size(), mpi_ttype,
inProcs[ii], tag2, comm) );
}
//our final communication operation is to catch the Irecvs we started above.
for(int ii=0; ii<numOutProcs; ++ii) {
MPI_Wait(&requests[ii], &statuses[ii]);
}
//now we've completed all the communication, so we're ready to put the data
//we received into the outPattern object.
offset = 0;
for(int ii=0; ii<numOutProcs; ii++) {
outPattern->addIndices(outProcs[ii], lengths[ii],
&(recvData[offset]));
offset += lengths[ii];
}
#endif
return(0);
}
