void MPICommunicator::establishConnectionChannel(){
// This function will open a port and publish a service in the name CS(control script).
MPI_Status status;
MPI_Open_port(MPI_INFO_NULL, &portname[0]);
// Publish the service
MPI_Publish_name((char*)(name.c_str()), MPI_INFO_NULL, &portname[0]);
}
MPI_Comm server_init( MPI_Comm comm )
{
char port_name[ MPI_MAX_PORT_NAME ];
MPI_Comm newcomm;
MPI_Open_port( MPI_INFO_NULL, port_name );
fprintf( stdout, "server: port opened at %s\n", port_name );
MPI_Publish_name( "mpe_port_name", MPI_INFO_NULL, port_name );
MPI_Comm_accept( port_name, MPI_INFO_NULL, 0, comm, &newcomm );
return newcomm;
}
void mpi_open_port_f(MPI_Fint *info, char *port_name, MPI_Fint *ierr, int port_name_len)
{
MPI_Info c_info;
char c_port_name[MPI_MAX_PORT_NAME];
c_info = MPI_Info_f2c(*info);
*ierr = OMPI_INT_2_FINT(MPI_Open_port(c_info, c_port_name));
if ( MPI_SUCCESS == OMPI_FINT_2_INT (*ierr )) {
ompi_fortran_string_c2f(c_port_name, port_name, port_name_len );
}
}
int main(int argc, char **argv)
{
int comm_rank;
char port_name[MPI_MAX_PORT_NAME];
MPI_Comm intercomm;
int ok_flag;
MPI_Init(&argc, &argv);
MPI_Comm_rank(MPI_COMM_WORLD, &comm_rank);
ok_flag = (comm_rank != 0) || (argc == 1);
MPI_Bcast(&ok_flag, 1, MPI_INT, 0, MPI_COMM_WORLD);
if (!ok_flag) {
if (comm_rank == 0) {
fprintf(stderr,"Usage: %s\n",argv[0]);
}
MPI_Abort(MPI_COMM_WORLD, 1);
}
MPI_Open_port(MPI_INFO_NULL, port_name);
if (comm_rank == 0) {
printf("Server port = '%s'\n", port_name);
}
MPI_Comm_accept(port_name, MPI_INFO_NULL, 0, MPI_COMM_WORLD, &intercomm);
MPI_Close_port(port_name);
if (comm_rank == 0) {
printf("MPI_Comm_accept() sucessful...\n");
}
MPI_Comm_disconnect(&intercomm);
MPI_Finalize();
return EXIT_SUCCESS;
}
void MPIPortsCommunication:: acceptConnection
(
const std::string& nameAcceptor,
const std::string& nameRequester,
int acceptorProcessRank,
int acceptorCommunicatorSize )
{
preciceTrace2 ( "acceptConnection()", nameAcceptor, nameRequester );
assertion ( not _isConnection );
int argc = 1;
char* arg = new char[8];
strcpy(arg, "precice");
char** argv = &arg;
utils::Parallel::initialize(&argc, &argv, nameAcceptor);
delete[] arg;
MPI_Open_port(MPI_INFO_NULL, _portname);
// Write portname to file
std::string portFilename ( _publishingDirectory + "." + nameRequester + "-portname" );
preciceDebug ( "Writing server connection info to file " + portFilename );
std::ofstream outFile;
outFile.open ((portFilename + "~").c_str(), std::ios::out);
outFile << _portname;
outFile.close();
// To give the file first a "wrong" name prevents early reading errors
rename( (portFilename + "~").c_str(), portFilename.c_str() );
preciceDebug("Calling MPI_Comm_accept() with portname = " << _portname);
MPI_Comm localComm = utils::Parallel::getLocalCommunicator();
MPI_Comm_accept ( _portname, MPI_INFO_NULL, 0, localComm, &communicator() );
if ( utils::Parallel::getLocalProcessRank() == 0 ){
if ( remove(portFilename.c_str()) != 0 ) {
preciceWarning ( "acceptConnection()", "Could not remove port information file!" );
}
}
_isConnection = true;
}
int main(int argc, char *argv[])
{
int error;
int rank, size;
char *argv1[2] = { (char*)"connector", NULL };
char *argv2[2] = { (char*)"acceptor", NULL };
MPI_Comm comm_connector, comm_acceptor, comm_parent, comm;
char port[MPI_MAX_PORT_NAME];
MPI_Status status;
MPI_Info spawn_path = MPI_INFO_NULL;
int verbose = 0;
if (getenv("MPITEST_VERBOSE"))
{
verbose = 1;
}
IF_VERBOSE(("init.\n"));
error = MPI_Init(&argc, &argv);
check_error(error, "MPI_Init");
/* To improve reporting of problems about operations, we
change the error handler to errors return */
MPI_Comm_set_errhandler( MPI_COMM_WORLD, MPI_ERRORS_RETURN );
MPI_Comm_set_errhandler( MPI_COMM_SELF, MPI_ERRORS_RETURN );
IF_VERBOSE(("size.\n"));
error = MPI_Comm_size(MPI_COMM_WORLD, &size);
check_error(error, "MPI_Comm_size");
IF_VERBOSE(("rank.\n"));
error = MPI_Comm_rank(MPI_COMM_WORLD, &rank);
check_error(error, "MPI_Comm_rank");
if (argc == 1)
{
/* Make sure that the current directory is in the path.
Not all implementations may honor or understand this, but
it is highly recommended as it gives users a clean way
to specify the location of the executable without
specifying a particular directory format (e.g., this
should work with both Windows and Unix implementations) */
error = MPI_Info_create( &spawn_path );
check_error( error, "MPI_Info_create" );
error = MPI_Info_set( spawn_path, (char*)"path", (char*)"." );
check_error( error, "MPI_Info_set" );
IF_VERBOSE(("spawn connector.\n"));
error = MPI_Comm_spawn((char*)"spaconacc", argv1, 1, spawn_path, 0,
MPI_COMM_SELF, &comm_connector,
MPI_ERRCODES_IGNORE);
check_error(error, "MPI_Comm_spawn");
IF_VERBOSE(("spawn acceptor.\n"));
error = MPI_Comm_spawn((char*)"spaconacc", argv2, 1, spawn_path, 0,
MPI_COMM_SELF, &comm_acceptor,
MPI_ERRCODES_IGNORE);
check_error(error, "MPI_Comm_spawn");
error = MPI_Info_free( &spawn_path );
check_error( error, "MPI_Info_free" );
MPI_Comm_set_errhandler( comm_connector, MPI_ERRORS_RETURN );
MPI_Comm_set_errhandler( comm_acceptor, MPI_ERRORS_RETURN );
IF_VERBOSE(("recv port.\n"));
error = MPI_Recv(port, MPI_MAX_PORT_NAME, MPI_CHAR, 0, 0,
comm_acceptor, &status);
check_error(error, "MPI_Recv");
IF_VERBOSE(("send port.\n"));
error = MPI_Send(port, MPI_MAX_PORT_NAME, MPI_CHAR, 0, 0,
comm_connector);
check_error(error, "MPI_Send");
IF_VERBOSE(("barrier acceptor.\n"));
error = MPI_Barrier(comm_acceptor);
check_error(error, "MPI_Barrier");
IF_VERBOSE(("barrier connector.\n"));
error = MPI_Barrier(comm_connector);
check_error(error, "MPI_Barrier");
error = MPI_Comm_free(&comm_acceptor);
check_error(error, "MPI_Comm_free");
error = MPI_Comm_free(&comm_connector);
check_error(error, "MPI_Comm_free");
printf(" No Errors\n");
}
else if ((argc == 2) && (strcmp(argv[1], "acceptor") == 0))
{
IF_VERBOSE(("get_parent.\n"));
error = MPI_Comm_get_parent(&comm_parent);
check_error(error, "MPI_Comm_get_parent");
if (comm_parent == MPI_COMM_NULL)
{
printf("acceptor's parent is NULL.\n");fflush(stdout);
MPI_Abort(MPI_COMM_WORLD, -1);
}
IF_VERBOSE(("open_port.\n"));
error = MPI_Open_port(MPI_INFO_NULL, port);
check_error(error, "MPI_Open_port");
MPI_Comm_set_errhandler( comm_parent, MPI_ERRORS_RETURN );
IF_VERBOSE(("0: opened port: <%s>\n", port));
IF_VERBOSE(("send.\n"));
error = MPI_Send(port, MPI_MAX_PORT_NAME, MPI_CHAR, 0, 0, comm_parent);
check_error(error, "MPI_Send");
IF_VERBOSE(("accept.\n"));
error = MPI_Comm_accept(port, MPI_INFO_NULL, 0, MPI_COMM_SELF, &comm);
check_error(error, "MPI_Comm_accept");
IF_VERBOSE(("close_port.\n"));
error = MPI_Close_port(port);
check_error(error, "MPI_Close_port");
IF_VERBOSE(("disconnect.\n"));
error = MPI_Comm_disconnect(&comm);
check_error(error, "MPI_Comm_disconnect");
IF_VERBOSE(("barrier.\n"));
error = MPI_Barrier(comm_parent);
check_error(error, "MPI_Barrier");
MPI_Comm_free( &comm_parent );
}
else if ((argc == 2) && (strcmp(argv[1], "connector") == 0))
{
IF_VERBOSE(("get_parent.\n"));
error = MPI_Comm_get_parent(&comm_parent);
check_error(error, "MPI_Comm_get_parent");
if (comm_parent == MPI_COMM_NULL)
{
printf("acceptor's parent is NULL.\n");fflush(stdout);
MPI_Abort(MPI_COMM_WORLD, -1);
}
MPI_Comm_set_errhandler( comm_parent, MPI_ERRORS_RETURN );
IF_VERBOSE(("recv.\n"));
error = MPI_Recv(port, MPI_MAX_PORT_NAME, MPI_CHAR, 0, 0,
comm_parent, &status);
check_error(error, "MPI_Recv");
IF_VERBOSE(("1: received port: <%s>\n", port));
IF_VERBOSE(("connect.\n"));
error = MPI_Comm_connect(port, MPI_INFO_NULL, 0, MPI_COMM_SELF, &comm);
check_error(error, "MPI_Comm_connect");
MPI_Comm_set_errhandler( comm, MPI_ERRORS_RETURN );
IF_VERBOSE(("disconnect.\n"));
error = MPI_Comm_disconnect(&comm);
check_error(error, "MPI_Comm_disconnect");
IF_VERBOSE(("barrier.\n"));
error = MPI_Barrier(comm_parent);
check_error(error, "MPI_Barrier");
MPI_Comm_free( &comm_parent );
}
else
{
printf("invalid command line.\n");fflush(stdout);
{
int i;
for (i=0; i<argc; i++)
{
printf("argv[%d] = <%s>\n", i, argv[i]);
}
}
fflush(stdout);
MPI_Abort(MPI_COMM_WORLD, -2);
}
MPI_Finalize();
return 0;
}
int main( int argc, char ** argv ) {
MPI_Comm tmp, comm, startComm;
char * fname;
char * actualFname = NULL;
char * globalFname = NULL;
int totalSize, expectedRank, size, cachedRank;
char portName[MPI_MAX_PORT_NAME];
int rankToAccept = 1;
/* Debug - print out where we picked up the MPICH build from */
#ifdef MPICHLIBSTR
msg( "MPICH library taken from: %s\n", MPICHLIBSTR );
#endif
if( argc != 4 ) {
printf( "Usage: %s <fname> <totalSize> <idx-1-based>\n", argv[0] );
exit( 1 );
}
/* This is the base name of the file into which we write the port */
fname = argv[1];
/* This is the total number of processes launched */
totalSize = atoi( argv[2] );
/* Each process knows its expected rank */
expectedRank = atoi( argv[3] )-1;
/* Start a watchdog thread which will abort after 120 seconds, and will
* print stack traces using GDB every 5 seconds if you don't call
* strokeWatchdog() */
startWatchdog( 120 );
/* Print a debug header */
msg( "Waiting for: %d - my rank is %d\n", totalSize, expectedRank );
/* Singleton init */
MPI_Init( 0, 0 );
/* Duplicate from MPI_COMM_SELF the starting point */
MPI_Comm_dup( MPI_COMM_SELF, &startComm );
if( expectedRank == 0 ) {
/* This process opens the port, and writes the information to the file */
MPI_Open_port( MPI_INFO_NULL, portName );
/* Write the port to fname.<rank> so that the connecting processes can
* wait their turn by checking for the correct file to show up */
actualFname = writePortToFile( portName, "%s.%d", fname, rankToAccept++ );
/* The wrapper script I'm using checks for the existance of "fname", so
* create that - even though it isn't used */
globalFname = writePortToFile( portName, fname );
installExitHandler( globalFname );
comm = startComm;
} else {
char * readPort;
readPort = getPortFromFile( "%s.%d", fname, expectedRank );
strncpy( portName, readPort, MPI_MAX_PORT_NAME );
free( readPort );
msg( "Read port <%s>\n", portName );
MPI_Comm_connect( portName, MPI_INFO_NULL, 0, startComm, &comm );
MPI_Intercomm_merge( comm, 1, &tmp );
comm = tmp;
MPI_Comm_size( comm, &size );
msg( "After my first merge, size is now: %d\n", size );
}
while( size < totalSize ) {
/* Make sure we don't print a stack until we stall */
strokeWatchdog();
/* Accept the connection */
MPI_Comm_accept( portName, MPI_INFO_NULL, 0, comm, &tmp );
/* Merge into intracomm */
MPI_Intercomm_merge( tmp, 0, &comm );
/* Free the intercomm */
MPI_Comm_free( &tmp );
/* See where we're up to */
MPI_Comm_rank( comm, &cachedRank );
MPI_Comm_size( comm, &size );
if( expectedRank == 0 ) {
msg( "Up to size: %d\n", size );
/* Delete the old file, create the new one */
unlink( actualFname );
free( actualFname );
/* Allow the next rank to connect */
actualFname = writePortToFile( portName, "%s.%d", fname, rankToAccept++ );
}
}
MPI_Comm_rank( comm, &cachedRank );
msg( "All done - I got rank: %d.\n", cachedRank );
MPI_Barrier( comm );
if( expectedRank == 0 ) {
/* Cleanup on rank zero - delete some files */
sleep( 4 );
unlink( actualFname );
free( actualFname );
unlink( globalFname );
free( globalFname );
/* This lets my wrapper script know that we did everything correctly */
indicateConnectSucceeded();
}
MPI_Finalize();
return 0;
}
int main(int argc, char *argv[])
{
char hostname[255] ;
char buff[255] ;
int role ;
int num_clients ;
int size, rank ;
FILE *fp ;
char server_port_name[MPI_MAX_PORT_NAME] ;
MPI_Comm intercomm, intracomm ;
MPI_Status status ;
int msg_count ;
int i ;
/* sanity check the args */
if(argc != 3)
{
fprintf(stderr, "usage %s <num clients> <1:server | 0:client>\n", argv[0]) ;
exit(1) ;
}
num_clients = atoi(argv[1]) ;
role = atoi(argv[2]) ;
if (num_clients <= 0 || (role != 0 && role != 1))
{
fprintf(stderr, "usage %s <num clients> <1:server | 0:client>\n", argv[0]) ;
exit(1) ;
}
/* initialize MPI */
CHK(MPI_Init(&argc, &argv)) ;
/* get the node name */
{
int retval = gethostname(hostname, 255) ;
if(retval == -1)
{
fprintf(stderr, "gethostname failed: %s\n", strerror(errno)) ;
exit(1) ;
}
}
/* server */
if(role == 1)
{
printf("SERVER: on node '%s'\n", hostname) ;
/* open port to establish connections */
CHK(MPI_Open_port(MPI_INFO_NULL, server_port_name)) ;
printf("SERVER: opened port=%s\n", server_port_name) ;
/* store the port name */
fp = fopen("server_port_name.txt", "w") ;
if(fp == NULL)
{
fprintf(stderr, "fopen failed: %s\n", strerror(errno)) ;
exit(1) ;
}
fprintf(fp, "%s", server_port_name) ;
fclose(fp) ;
/* the server accepts connections from all the clients */
for(i = 0 ; i < num_clients ; i++ )
{
/* accept connections at this port */
CHK(MPI_Comm_accept(server_port_name, MPI_INFO_NULL, 0,
i == 0 ? MPI_COMM_WORLD : intracomm,
&intercomm)) ;
printf("SERVER: accepted connection from client %d\n", i+1) ;
/* merge, to form one intra communicator */
CHK(MPI_Intercomm_merge(intercomm, 0, &intracomm)) ;
printf("SERVER: merged with client %d\n", i+1) ;
CHK(MPI_Comm_size(intracomm, &size)) ;
CHK(MPI_Comm_rank(intracomm, &rank)) ;
printf("SERVER: after merging with client %d: size=%d rank=%d\n", i+1, size, rank) ;
}
} /* end server */
/* client */
if(role == 0)
{
printf("CLIENT: on node '%s'\n", hostname) ;
fp = fopen("server_port_name.txt", "r") ;
if(fp == NULL)
{
fprintf(stderr, "fopen failed: %s\n", strerror(errno)) ;
exit(1) ;
}
fscanf(fp, "%s", server_port_name) ;
fclose(fp) ;
printf("CLIENT: attempting to connect to server on port=%s\n", server_port_name) ;
/* connect to the server */
CHK(MPI_Comm_connect (server_port_name, MPI_INFO_NULL, 0, MPI_COMM_WORLD, &intercomm)) ;
printf("CLIENT: connected to server on port\n") ;
/* merge the server and client to one intra communicator */
CHK(MPI_Intercomm_merge(intercomm, 1, &intracomm)) ;
printf("CLIENT: merged with existing intracomm\n") ;
CHK(MPI_Comm_size(intracomm, &size)) ;
CHK(MPI_Comm_rank(intracomm, &rank)) ;
printf("CLIENT: after merging, new comm: size=%d rank=%d\n", size, rank) ;
for (i = rank ; i < num_clients ; i++)
{
/* client performs a collective accept */
CHK(MPI_Comm_accept(server_port_name, MPI_INFO_NULL, 0, intracomm, &intercomm)) ;
printf("CLIENT: connected to server on port\n") ;
/* merge the two intra comms back to one communicator */
CHK(MPI_Intercomm_merge(intercomm, 0, &intracomm)) ;
printf("CLIENT: merged with existing members\n") ;
CHK(MPI_Comm_size(intracomm, &size)) ;
CHK(MPI_Comm_rank(intracomm, &rank)) ;
printf("CLIENT: new size after merging with existing members: size=%d rank=%d\n", size, rank) ;
}
} /* end client */
CHK(MPI_Comm_size(intracomm, &size)) ;
CHK(MPI_Comm_rank(intracomm, &rank)) ;
printf("After fusion: size=%d rank=%d\n", size, rank) ;
if(rank == 0)
{
msg_count = num_clients ;
while(msg_count)
{
CHK(MPI_Recv(buff, 255, MPI_CHAR, MPI_ANY_SOURCE,
MPI_ANY_TAG, intracomm, &status)) ;
printf("Received hello msg from '%s'\n", buff) ;
msg_count-- ;
}
}
else
{
/* all ranks > 0 */
CHK(MPI_Send(hostname, strlen(hostname) + 1, MPI_CHAR, 0, TAG, intracomm)) ;
}
CHK(MPI_Finalize()) ;
fprintf(stderr, "Rank %d is exiting\n", rank);
return 0 ;
}
/* This test spawns two child jobs and has them open a port and connect to
* each other.
* The two children repeatedly connect, accept, and disconnect from each other.
*/
int main(int argc, char *argv[])
{
int error;
int rank, size;
int numprocs = 3;
char *argv1[2] = { (char *) "connector", NULL };
char *argv2[2] = { (char *) "acceptor", NULL };
MPI_Comm comm_connector, comm_acceptor, comm_parent, comm;
char port[MPI_MAX_PORT_NAME] = { 0 };
MPI_Status status;
MPI_Info spawn_path = MPI_INFO_NULL;
int i, num_loops = 100;
int data;
int verbose = 0;
int can_spawn, errs = 0;
if (getenv("MPITEST_VERBOSE")) {
verbose = 1;
}
IF_VERBOSE(("init.\n"));
error = MPI_Init(&argc, &argv);
check_error(error, "MPI_Init");
errs += MTestSpawnPossible(&can_spawn);
if (!can_spawn) {
if (errs)
printf(" Found %d errors\n", errs);
else
printf(" No Errors\n");
fflush(stdout);
} else {
IF_VERBOSE(("size.\n"));
error = MPI_Comm_size(MPI_COMM_WORLD, &size);
check_error(error, "MPI_Comm_size");
IF_VERBOSE(("rank.\n"));
error = MPI_Comm_rank(MPI_COMM_WORLD, &rank);
check_error(error, "MPI_Comm_rank");
if (argc == 1) {
/* Make sure that the current directory is in the path.
* Not all implementations may honor or understand this, but
* it is highly recommended as it gives users a clean way
* to specify the location of the executable without
* specifying a particular directory format (e.g., this
* should work with both Windows and Unix implementations) */
MPI_Info_create(&spawn_path);
MPI_Info_set(spawn_path, (char *) "path", (char *) ".");
IF_VERBOSE(("spawn connector.\n"));
error = MPI_Comm_spawn((char *) "disconnect_reconnect2",
argv1, numprocs, spawn_path, 0,
MPI_COMM_WORLD, &comm_connector, MPI_ERRCODES_IGNORE);
check_error(error, "MPI_Comm_spawn");
IF_VERBOSE(("spawn acceptor.\n"));
error = MPI_Comm_spawn((char *) "disconnect_reconnect2",
argv2, numprocs, spawn_path, 0,
MPI_COMM_WORLD, &comm_acceptor, MPI_ERRCODES_IGNORE);
check_error(error, "MPI_Comm_spawn");
MPI_Info_free(&spawn_path);
if (rank == 0) {
IF_VERBOSE(("recv port.\n"));
error = MPI_Recv(port, MPI_MAX_PORT_NAME, MPI_CHAR, 0, 0, comm_acceptor, &status);
check_error(error, "MPI_Recv");
IF_VERBOSE(("send port.\n"));
error = MPI_Send(port, MPI_MAX_PORT_NAME, MPI_CHAR, 0, 0, comm_connector);
check_error(error, "MPI_Send");
}
IF_VERBOSE(("barrier acceptor.\n"));
error = MPI_Barrier(comm_acceptor);
check_error(error, "MPI_Barrier");
IF_VERBOSE(("barrier connector.\n"));
error = MPI_Barrier(comm_connector);
check_error(error, "MPI_Barrier");
error = MPI_Comm_free(&comm_acceptor);
check_error(error, "MPI_Comm_free");
error = MPI_Comm_free(&comm_connector);
check_error(error, "MPI_Comm_free");
if (rank == 0) {
printf(" No Errors\n");
fflush(stdout);
}
} else if ((argc == 2) && (strcmp(argv[1], "acceptor") == 0)) {
IF_VERBOSE(("get_parent.\n"));
error = MPI_Comm_get_parent(&comm_parent);
check_error(error, "MPI_Comm_get_parent");
if (comm_parent == MPI_COMM_NULL) {
printf("acceptor's parent is NULL.\n");
fflush(stdout);
MPI_Abort(MPI_COMM_WORLD, -1);
}
if (rank == 0) {
IF_VERBOSE(("open_port.\n"));
error = MPI_Open_port(MPI_INFO_NULL, port);
check_error(error, "MPI_Open_port");
IF_VERBOSE(("0: opened port: <%s>\n", port));
IF_VERBOSE(("send.\n"));
error = MPI_Send(port, MPI_MAX_PORT_NAME, MPI_CHAR, 0, 0, comm_parent);
check_error(error, "MPI_Send");
}
for (i = 0; i < num_loops; i++) {
IF_VERBOSE(("accept.\n"));
error = MPI_Comm_accept(port, MPI_INFO_NULL, 0, MPI_COMM_WORLD, &comm);
check_error(error, "MPI_Comm_accept");
if (rank == 0) {
data = i;
error = MPI_Send(&data, 1, MPI_INT, 0, 0, comm);
check_error(error, "MPI_Send");
error = MPI_Recv(&data, 1, MPI_INT, 0, 0, comm, &status);
check_error(error, "MPI_Recv");
if (data != i) {
printf("expected %d but received %d\n", i, data);
fflush(stdout);
MPI_Abort(MPI_COMM_WORLD, 1);
}
}
IF_VERBOSE(("disconnect.\n"));
error = MPI_Comm_disconnect(&comm);
check_error(error, "MPI_Comm_disconnect");
}
if (rank == 0) {
IF_VERBOSE(("close_port.\n"));
error = MPI_Close_port(port);
check_error(error, "MPI_Close_port");
}
IF_VERBOSE(("barrier.\n"));
error = MPI_Barrier(comm_parent);
check_error(error, "MPI_Barrier");
MPI_Comm_free(&comm_parent);
} else if ((argc == 2) && (strcmp(argv[1], "connector") == 0)) {
IF_VERBOSE(("get_parent.\n"));
error = MPI_Comm_get_parent(&comm_parent);
check_error(error, "MPI_Comm_get_parent");
if (comm_parent == MPI_COMM_NULL) {
printf("acceptor's parent is NULL.\n");
fflush(stdout);
MPI_Abort(MPI_COMM_WORLD, -1);
}
if (rank == 0) {
IF_VERBOSE(("recv.\n"));
error = MPI_Recv(port, MPI_MAX_PORT_NAME, MPI_CHAR, 0, 0, comm_parent, &status);
check_error(error, "MPI_Recv");
IF_VERBOSE(("1: received port: <%s>\n", port));
}
for (i = 0; i < num_loops; i++) {
IF_VERBOSE(("connect.\n"));
error = MPI_Comm_connect(port, MPI_INFO_NULL, 0, MPI_COMM_WORLD, &comm);
check_error(error, "MPI_Comm_connect");
if (rank == 0) {
data = -1;
error = MPI_Recv(&data, 1, MPI_INT, 0, 0, comm, &status);
check_error(error, "MPI_Recv");
if (data != i) {
printf("expected %d but received %d\n", i, data);
fflush(stdout);
MPI_Abort(MPI_COMM_WORLD, 1);
}
error = MPI_Send(&data, 1, MPI_INT, 0, 0, comm);
check_error(error, "MPI_Send");
}
IF_VERBOSE(("disconnect.\n"));
error = MPI_Comm_disconnect(&comm);
check_error(error, "MPI_Comm_disconnect");
}
IF_VERBOSE(("barrier.\n"));
error = MPI_Barrier(comm_parent);
check_error(error, "MPI_Barrier");
MPI_Comm_free(&comm_parent);
} else {
printf("invalid command line.\n");
fflush(stdout);
{
int ii;
for (ii = 0; ii < argc; ii++) {
printf("argv[%d] = <%s>\n", ii, argv[ii]);
}
}
fflush(stdout);
MPI_Abort(MPI_COMM_WORLD, -2);
}
}
MPI_Finalize();
return MTestReturnValue(errs);
}
int main( int argc, char *argv[] )
{
int num_errors = 0, total_num_errors = 0;
int rank, size;
char port1[MPI_MAX_PORT_NAME];
char port2[MPI_MAX_PORT_NAME];
char port3[MPI_MAX_PORT_NAME];
MPI_Status status;
MPI_Comm comm1, comm2, comm3;
int verbose = 0;
int data = 0;
if (getenv("MPITEST_VERBOSE"))
{
verbose = 1;
}
MPI_Init(&argc, &argv);
MPI_Comm_size(MPI_COMM_WORLD, &size);
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
if (size < 4)
{
printf("Four processes needed to run this test.\n");
MPI_Finalize();
return 0;
}
if (rank == 0)
{
IF_VERBOSE(("0: opening ports.\n"));
MPI_Open_port(MPI_INFO_NULL, port1);
MPI_Open_port(MPI_INFO_NULL, port2);
MPI_Open_port(MPI_INFO_NULL, port3);
IF_VERBOSE(("0: opened port1: <%s>\n", port1));
IF_VERBOSE(("0: opened port2: <%s>\n", port2));
IF_VERBOSE(("0: opened port3: <%s>\n", port3));
IF_VERBOSE(("0: sending ports.\n"));
MPI_Send(port1, MPI_MAX_PORT_NAME, MPI_CHAR, 1, 0, MPI_COMM_WORLD);
MPI_Send(port2, MPI_MAX_PORT_NAME, MPI_CHAR, 2, 0, MPI_COMM_WORLD);
MPI_Send(port3, MPI_MAX_PORT_NAME, MPI_CHAR, 3, 0, MPI_COMM_WORLD);
IF_VERBOSE(("0: accepting port3.\n"));
MPI_Comm_accept(port3, MPI_INFO_NULL, 0, MPI_COMM_SELF, &comm3);
IF_VERBOSE(("0: accepting port2.\n"));
MPI_Comm_accept(port2, MPI_INFO_NULL, 0, MPI_COMM_SELF, &comm2);
IF_VERBOSE(("0: accepting port1.\n"));
MPI_Comm_accept(port1, MPI_INFO_NULL, 0, MPI_COMM_SELF, &comm1);
IF_VERBOSE(("0: closing ports.\n"));
MPI_Close_port(port1);
MPI_Close_port(port2);
MPI_Close_port(port3);
IF_VERBOSE(("0: sending 1 to process 1.\n"));
data = 1;
MPI_Send(&data, 1, MPI_INT, 0, 0, comm1);
IF_VERBOSE(("0: sending 2 to process 2.\n"));
data = 2;
MPI_Send(&data, 1, MPI_INT, 0, 0, comm2);
IF_VERBOSE(("0: sending 3 to process 3.\n"));
data = 3;
MPI_Send(&data, 1, MPI_INT, 0, 0, comm3);
IF_VERBOSE(("0: disconnecting.\n"));
MPI_Comm_disconnect(&comm1);
MPI_Comm_disconnect(&comm2);
MPI_Comm_disconnect(&comm3);
}
else if (rank == 1)
{
IF_VERBOSE(("1: receiving port.\n"));
MPI_Recv(port1, MPI_MAX_PORT_NAME, MPI_CHAR, 0, 0, MPI_COMM_WORLD, &status);
IF_VERBOSE(("1: received port1: <%s>\n", port1));
IF_VERBOSE(("1: connecting.\n"));
MPI_Comm_connect(port1, MPI_INFO_NULL, 0, MPI_COMM_SELF, &comm1);
MPI_Recv(&data, 1, MPI_INT, 0, 0, comm1, &status);
if (data != 1)
{
printf("Received %d from root when expecting 1\n", data);
fflush(stdout);
num_errors++;
}
IF_VERBOSE(("1: disconnecting.\n"));
MPI_Comm_disconnect(&comm1);
}
else if (rank == 2)
{
IF_VERBOSE(("2: receiving port.\n"));
MPI_Recv(port2, MPI_MAX_PORT_NAME, MPI_CHAR, 0, 0, MPI_COMM_WORLD, &status);
IF_VERBOSE(("2: received port2: <%s>\n", port2));
/* make sure process 1 has time to do the connect before this process
attempts to connect */
MTestSleep(2);
IF_VERBOSE(("2: connecting.\n"));
MPI_Comm_connect(port2, MPI_INFO_NULL, 0, MPI_COMM_SELF, &comm2);
MPI_Recv(&data, 1, MPI_INT, 0, 0, comm2, &status);
if (data != 2)
{
printf("Received %d from root when expecting 2\n", data);
fflush(stdout);
num_errors++;
}
IF_VERBOSE(("2: disconnecting.\n"));
MPI_Comm_disconnect(&comm2);
}
else if (rank == 3)
{
IF_VERBOSE(("3: receiving port.\n"));
MPI_Recv(port3, MPI_MAX_PORT_NAME, MPI_CHAR, 0, 0, MPI_COMM_WORLD, &status);
IF_VERBOSE(("2: received port2: <%s>\n", port2));
/* make sure process 1 and 2 have time to do the connect before this
process attempts to connect */
MTestSleep(4);
IF_VERBOSE(("3: connecting.\n"));
MPI_Comm_connect(port3, MPI_INFO_NULL, 0, MPI_COMM_SELF, &comm3);
MPI_Recv(&data, 1, MPI_INT, 0, 0, comm3, &status);
if (data != 3)
{
printf("Received %d from root when expecting 3\n", data);
fflush(stdout);
num_errors++;
}
IF_VERBOSE(("3: disconnecting.\n"));
MPI_Comm_disconnect(&comm3);
}
MPI_Barrier(MPI_COMM_WORLD);
MPI_Reduce(&num_errors, &total_num_errors, 1, MPI_INT, MPI_SUM, 0, MPI_COMM_WORLD);
if (rank == 0)
{
if (total_num_errors)
{
printf(" Found %d errors\n", total_num_errors);
}
else
{
printf(" No Errors\n");
}
fflush(stdout);
}
MPI_Finalize();
return total_num_errors;
}
int main(int argc, char *argv[])
{
int errs = 0;
int rank, size, rsize, i, j, data, num_loops = 100;
int np = 3;
MPI_Comm parentcomm, intercomm;
MPI_Status status;
char port[MPI_MAX_PORT_NAME] = {0};
int verbose = 0;
int do_messages = 1;
char *env;
env = getenv("MPITEST_VERBOSE");
if (env)
{
if (*env != '0')
verbose = 1;
}
MTest_Init( &argc, &argv );
/* FIXME: Document arguments */
if (argc > 1) {
num_loops = atoi(argv[1]);
if (num_loops < 0)
num_loops = 0;
if (num_loops > 100)
num_loops = 100;
}
if (argc > 2)
{
do_messages = atoi(argv[2]);
}
MPI_Comm_get_parent( &parentcomm );
if (parentcomm == MPI_COMM_NULL)
{
MPI_Comm_rank( MPI_COMM_WORLD, &rank ); /* Get rank for verbose msg */
IF_VERBOSE(("[%d] spawning %d processes\n", rank, np));
/* Create 3 more processes */
MPI_Comm_spawn((char*)"./disconnect_reconnect",
/*MPI_ARGV_NULL*/&argv[1], np,
MPI_INFO_NULL, 0, MPI_COMM_WORLD,
&intercomm, MPI_ERRCODES_IGNORE);
}
else
{
intercomm = parentcomm;
}
/* We now have a valid intercomm */
MPI_Comm_remote_size(intercomm, &rsize);
MPI_Comm_size(intercomm, &size);
MPI_Comm_rank(intercomm, &rank);
if (parentcomm == MPI_COMM_NULL)
{
IF_VERBOSE(("[%d] parent rank %d alive.\n", rank, rank));
/* Parent */
if (rsize != np)
{
errs++;
printf("Did not create %d processes (got %d)\n", np, rsize);
fflush(stdout);
}
if (rank == 0 && num_loops > 0)
{
MPI_Open_port(MPI_INFO_NULL, port);
IF_VERBOSE(("[%d] port = %s\n", rank, port));
MPI_Send(port, MPI_MAX_PORT_NAME, MPI_CHAR, 0, 0, intercomm);
}
IF_VERBOSE(("[%d] disconnecting child communicator\n",rank));
MPI_Comm_disconnect(&intercomm);
for (i=0; i<num_loops; i++)
{
IF_VERBOSE(("[%d] accepting connection\n",rank));
MPI_Comm_accept(port, MPI_INFO_NULL, 0, MPI_COMM_WORLD, &intercomm);
MPI_Comm_remote_size(intercomm, &rsize);
if (do_messages && (rank == 0))
{
j = 0;
for (j=0; j<rsize; j++)
{
data = i;
IF_VERBOSE(("[%d]sending int to child process %d\n", rank, j));
MPI_Send(&data, 1, MPI_INT, j, 100, intercomm);
IF_VERBOSE(("[%d] receiving int from child process %d\n", rank, j));
data = i-1;
MPI_Recv(&data, 1, MPI_INT, j, 100, intercomm, &status);
if (data != i)
{
errs++;
}
}
}
IF_VERBOSE(("[%d] disconnecting communicator\n", rank));
MPI_Comm_disconnect(&intercomm);
}
/* Errors cannot be sent back to the parent because there is no
communicator connected to the children
for (i=0; i<rsize; i++)
{
MPI_Recv( &err, 1, MPI_INT, i, 1, intercomm, MPI_STATUS_IGNORE );
errs += err;
}
*/
}
else
{
IF_VERBOSE(("[%d] child rank %d alive.\n", rank, rank));
/* Child */
if (size != np)
{
errs++;
printf("(Child) Did not create %d processes (got %d)\n", np, size);
fflush(stdout);
}
if (rank == 0 && num_loops > 0)
{
IF_VERBOSE(("[%d] receiving port\n", rank));
MPI_Recv(port, MPI_MAX_PORT_NAME, MPI_CHAR, 0, 0, intercomm, &status);
}
IF_VERBOSE(("[%d] disconnecting communicator\n", rank));
MPI_Comm_disconnect(&intercomm);
for (i=0; i<num_loops; i++)
{
IF_VERBOSE(("[%d] connecting to port (loop %d)\n",rank,i));
MPI_Comm_connect(port, MPI_INFO_NULL, 0, MPI_COMM_WORLD, &intercomm);
if (do_messages)
{
IF_VERBOSE(("[%d] receiving int from parent process 0\n",rank));
MPI_Recv(&data, 1, MPI_INT, 0, 100, intercomm, &status);
if (data != i)
{
printf("expected %d but received %d\n", i, data);
fflush(stdout);
MPI_Abort(MPI_COMM_WORLD, 1);
}
IF_VERBOSE(("[%d] sending int back to parent process 1\n",rank));
MPI_Send(&data, 1, MPI_INT, 0, 100, intercomm);
}
IF_VERBOSE(("[%d] disconnecting communicator\n",rank));
MPI_Comm_disconnect(&intercomm);
}
/* Send the errs back to the master process */
/* Errors cannot be sent back to the parent because there is no
communicator connected to the parent */
/*MPI_Ssend( &errs, 1, MPI_INT, 0, 1, intercomm );*/
}
/* Note that the MTest_Finalize get errs only over COMM_WORLD */
/* Note also that both the parent and child will generate "No Errors"
if both call MTest_Finalize */
if (parentcomm == MPI_COMM_NULL)
{
MTest_Finalize( errs );
}
IF_VERBOSE(("[%d] calling finalize\n",rank));
MPI_Finalize();
return 0;
}
#undef MPI_Open_port
#define MPI_Open_port PMPI_Open_port
#else
#ifdef F77_NAME_UPPER
#define mpi_open_port_ MPI_OPEN_PORT
#elif defined(F77_NAME_LOWER_2USCORE)
#define mpi_open_port_ mpi_open_port__
#elif !defined(F77_NAME_LOWER_USCORE)
#define mpi_open_port_ mpi_open_port
/* Else leave name alone */
#endif
#endif /* MPICH_MPI_FROM_PMPI */
/* Prototypes for the Fortran interfaces */
#include "fproto.h"
FORT_DLL_SPEC void FORT_CALL mpi_open_port_ ( MPI_Fint *v1, char *v2 FORT_MIXED_LEN(d2), MPI_Fint *ierr FORT_END_LEN(d2) ){
char *p2;
p2 = (char *)MPIU_Malloc( d2 + 1 );
*ierr = MPI_Open_port( (MPI_Info)(*v1), p2 );
if (!*ierr) {char *p = v2, *pc=p2;
while (*pc) {*p++ = *pc++;}
while ((p-v2) < d2) { *p++ = ' '; }
}
MPIU_Free( p2 );
}
void run_client(int size) {
int data;
MPI_Comm comm1, comm2;
MPI_Status status;
char port1[MPI_MAX_PORT_NAME];
char port2[MPI_MAX_PORT_NAME];
//rankprintf("opening ports.\n");fflush(stdout);
MPI_Open_port(MPI_INFO_NULL, port1);
MPI_Open_port(MPI_INFO_NULL, port2);
//rankprintf("opened port1: <%s>\n", port1);
//rankprintf("opened port2: <%s>\n", port2);fflush(stdout);
MPI_Send(port1, MPI_MAX_PORT_NAME, MPI_CHAR, 1, 0, MPI_COMM_WORLD);
MPI_Send(port2, MPI_MAX_PORT_NAME, MPI_CHAR, 2, 0, MPI_COMM_WORLD);
//rankprintf("accepting port2.\n");fflush(stdout);
MPI_Comm_accept(port2, MPI_INFO_NULL, 0, MPI_COMM_SELF, &comm2);
//rankprintf("accepting port1.\n");fflush(stdout);
MPI_Comm_accept(port1, MPI_INFO_NULL, 0, MPI_COMM_SELF, &comm1);
MPI_Close_port(port1);
MPI_Close_port(port2);
MPI_Barrier(MPI_COMM_WORLD);
debugMark(0);
//Ping both nodes
data = MSG_CHECK;
MPI_Send(&data, 1, MPI_INT, 0, 0, comm1);
MPI_Send(&data, 1, MPI_INT, 0, 0, comm2);
sleep(1);
MPI_Recv(&data, 1, MPI_INT, 0, 0, comm1, &status);
if (MSG_OK != data) {
sleep(3);
MPI_Recv(&data, 1, MPI_INT, 0, 0, comm1, &status);
}
if (MSG_OK != data) {
rankprintf("Rank 1 not OK\n");
} else {
rankprintf("Rank 1 OK\n");
}
MPI_Recv(&data, 1, MPI_INT, 0, 0, comm2, &status);
if (MSG_OK != data) {
sleep(3);
MPI_Recv(&data, 1, MPI_INT, 0, 0, comm2, &status);
}
if (MSG_OK != data) {
rankprintf("Rank 2 not OK\n");
} else {
rankprintf("Rank 2 OK\n");
}
//Create and fill two queues
taskQueue q1(comm1);
taskQueue q2(comm2);
q1.append(task(TASK_GENVIS, 45, NPOINTS));
q1.append(task(TASK_GENIMG, 46, IMG_SIZE));
q1.append(task(TASK_DEGRID, 45, 46));
q2.append(task(TASK_GENVIS, 47, NPOINTS));
q2.append(task(TASK_GENIMG, 48, IMG_SIZE));
q2.append(task(TASK_DEGRID, 47, 48));
//Submit tasks until both queues are empty
while (!(q1.empty() && q2.empty())) {
if (q1.done() && !q1.empty()) q1.send_next();
if (q2.done() && !q2.empty()) q2.send_next();
sleep(1);
}
//Wait for final tasks to complete
while (!(q1.done() && q2.done())) {
sleep(1);
}
rankprintf("Closing\n");
//Close connections
data = INT_MAX;
MPI_Send(&data, 1, MPI_INT, 0, 0, comm1);
MPI_Send(&data, 1, MPI_INT, 0, 0, comm2);
MPI_Comm_disconnect(&comm1);
MPI_Comm_disconnect(&comm2);
}
