int MPI_Init(int* argc, char*** argv)
{
parse_dynamic_server(getenv("EPLIB_DYNAMIC_SERVER"));
/* Initialize MPI */
/* Special handling for async thread */
int ret;
if (dynamic_server == DYNAMIC_SERVER_ASYNCTHREAD)
{
int provided;
ret = PMPI_Init_thread(argc, argv, MPI_THREAD_MULTIPLE, &provided);
if (provided != MPI_THREAD_MULTIPLE)
{
PRINT("Requested thread level not provided.\n");
PMPI_Abort(MPI_COMM_WORLD, -1);
}
}
else
ret = PMPI_Init(argc, argv);
/* Initialize EPLIB */
EPLIB_init();
return ret;
}
int MPI_Init( int *argc, char **argv[] )
{
int i, j;
// call real init function
PMPI_Init( argc, argv );
// get current rank
PMPI_Comm_rank(MPI_COMM_WORLD, &my_rank);
// get total number of process
PMPI_Comm_size(MPI_COMM_WORLD, &proc_num);
if(my_rank == 0){
send_count = (int *)malloc(proc_num * proc_num * sizeof(int));
send_size = (int *)malloc(proc_num * proc_num * sizeof(int));
}
// allocate space for local statistic storage
my_send_count = (int *)malloc(proc_num * sizeof(int));
my_send_size = (int *)malloc(proc_num * sizeof(int));
// init stastistic storage
for(i = 0; i < proc_num; i++){
my_send_count[i] = 0;
my_send_size[i] = 0;
}
return 0;
}
static int
_MPI_Init (int *argc, char ***argv)
{
int rc = 0;
int enabledStatus;
enabledStatus = mpiPi.enabled;
mpiPi.enabled = 0;
rc = PMPI_Init (argc, argv);
mpiPi.enabled = enabledStatus;
#if defined(Linux) && ! defined(ppc64)
mpiPi.appFullName = getProcExeLink ();
mpiPi_msg_debug ("appFullName is %s\n", mpiPi.appFullName);
mpiPi_init (GetBaseAppName (mpiPi.appFullName));
#else
if (argv != NULL && *argv != NULL && **argv != NULL)
{
mpiPi_init (GetBaseAppName (**argv));
mpiPi.appFullName = strdup (**argv);
}
else
{
mpiPi_init ("Unknown");
mpiPi_msg_debug ("argv is NULL\n");
}
#endif
return rc;
}
/* Use the Profile feature of MPI to call the pause_proc() */
int MPI_Init(int *argc, char ***argv)
{
int ret_code;
ret_code=PMPI_Init(argc, argv);
pause_proc();
return (ret_code);
}
int MPI_Init(int* argc, char*** argv)
{
int result, MPIT_result;
int provided;
result = PMPI_Init(argc, argv);
PMPI_Comm_size(MPI_COMM_WORLD, &comm_world_size);
PMPI_Comm_rank(MPI_COMM_WORLD, &comm_world_rank);
MPIT_result = MPI_T_init_thread(MPI_THREAD_SINGLE, &provided);
if (MPIT_result != MPI_SUCCESS) {
fprintf(stderr, "ERROR : failed to intialize MPI_T interface, preventing to get monitoring results: check your OpenMPI installation\n");
PMPI_Abort(MPI_COMM_WORLD, MPIT_result);
}
MPIT_result = MPI_T_pvar_session_create(&session);
if (MPIT_result != MPI_SUCCESS) {
fprintf(stderr, "ERROR : failed to create MPI_T session, preventing to get monitoring results: check your OpenMPI installation\n");
PMPI_Abort(MPI_COMM_WORLD, MPIT_result);
}
init_monitoring_result("pml_monitoring_messages_count", &counts);
init_monitoring_result("pml_monitoring_messages_size", &sizes);
start_monitoring_result(&counts);
start_monitoring_result(&sizes);
return result;
}
int MPI_Init(int *argc, char ***argv)
{
int err;
PNMPI_modHandle_t handle;
PNMPI_Service_descriptor_t s_del,s_get;
/* call the init routines */
err=PMPI_Init(argc,argv);
if (err!=MPI_SUCCESS)
return err;
/* query the other modules */
err=PNMPI_Service_GetModuleByName(PNMPI_MODULE_DATATYPE,&handle);
if (err!=MPI_SUCCESS)
return err;
err=PNMPI_Service_GetServiceByName(handle,"getDatatypeReference","pimp",&s_get);
if (err!=MPI_SUCCESS)
return err;
r_get=(PNMPIMOD_Datatype_getReference_t) s_get.fct;
err=PNMPI_Service_GetServiceByName(handle,"delDatatypeReference","p",&s_del);
if (err!=MPI_SUCCESS)
return err;
r_del=(PNMPIMOD_Datatype_delReference_t) s_del.fct;
return err;
}
/*-----------------------------------------------------------------------------
* Function MPI_Init
* Purpose: override MPI initialization in C/C++ programs to capture MPI rank
*---------------------------------------------------------------------------*/
int MPI_Init( int *argc, char ***argv )
{
int retval = PMPI_Init(argc, argv);
if (retval == MPI_SUCCESS) remember_rank();
return retval;
}
int MPI_Init(int *argc, char*** argv)
{
int wynik=PMPI_Init(argc,argv);
MPE_Init_log();
int proc;
MPI_Comm_rank(MPI_COMM_WORLD,&proc);
if(proc==0)
{
MPE_Describe_state(START_BCAST,END_BCAST,"broadcast","red");
MPE_Describe_state(START_SEND,END_SEND,"send","blue");
MPE_Describe_state(START_RECV,END_RECV,"recv","green");
};
MPE_Start_log();
return wynik;
};
int __give_pebil_name(MPI_Init)(int* argc, char*** argv){
int retval = 0;
#else
int __wrapper_name(MPI_Init)(int* argc, char*** argv){
int retval = PMPI_Init(argc, argv);
#endif // USES_PSINSTRACER
PMPI_Comm_rank(MPI_COMM_WORLD, &__taskid);
PMPI_Comm_size(MPI_COMM_WORLD, &__ntasks);
mpiValid = 1;
fprintf(stdout, "-[p%d]- remapping to taskid %d/%d on host %u in MPI_Init wrapper\n", getpid(), __taskid, __ntasks, gethostid());
tool_mpi_init();
return retval;
}
int MPI_Init(int *argc, char ***argv)
{
int returnVal;
returnVal = PMPI_Init( argc, argv );
MPI_Comm_rank( MPI_COMM_WORLD, &procid_1 );
requests_head_1 = requests_tail_1 = 0;
rq_init( requests_avail_1 );
MPI_Comm_rank( MPI_COMM_WORLD, &procid_0 );
MPI_Comm_size( MPI_COMM_WORLD, &np_0 );
MPI_Barrier( MPI_COMM_WORLD );
DrawScreen_0( procid_0, np_0 );
return returnVal;
}
int NQJ_Init(int * _pnmpi_arg_0, char * * * _pnmpi_arg_1)
{
int res;
int start_level;
start_level=pnmpi_level;
if (IS_ACTIVATED(MPI_Init_MAJOR,MPI_Init_MINOR))
{
while ((pnmpi_level<pnmpi_max_level) && (modules.module[pnmpi_level]->stack_delimiter==0))
{
if (pnmpi_function_ptrs.pnmpi_int_MPI_Init[pnmpi_level]!=NULL)
{
DBGPRINT3("Calling a wrapper in MPI_Init at level %i FROM %px",pnmpi_level,&(Internal_XMPI_Init));
res=(pnmpi_function_ptrs.pnmpi_int_MPI_Init)[pnmpi_level](_pnmpi_arg_0, _pnmpi_arg_1);
DBGPRINT3("Done with wrapper in MPI_Init at level %i - reseting to %i",pnmpi_level,start_level);
pnmpi_level=start_level;
return res;
}
pnmpi_level++;
}
}
if (pnmpi_init_done)
{
DBGPRINT3("Duplicated: calling a original MPI in MPI_Init");
res=MPI_SUCCESS;
}
else
{
DBGPRINT3("Calling a original MPI in MPI_Init");
#ifdef COMPILE_FOR_FORTRAN
if (init_was_fortran)
pmpi_init_(&res);
else
#endif
res=PMPI_Init(_pnmpi_arg_0, _pnmpi_arg_1);
pnmpi_init_done=1;
}
DBGPRINT3("Done with original MPI in MPI_Init");
pnmpi_level=start_level;
return res;
}
void smpi_replay_init(int *argc, char***argv){
PMPI_Init(argc, argv);
if (!smpi_process_index()){
_xbt_replay_action_init();
xbt_replay_action_register("init", action_init);
xbt_replay_action_register("finalize", action_finalize);
xbt_replay_action_register("comm_size",action_comm_size);
xbt_replay_action_register("send", action_send);
xbt_replay_action_register("Isend", action_Isend);
xbt_replay_action_register("recv", action_recv);
xbt_replay_action_register("Irecv", action_Irecv);
xbt_replay_action_register("wait", action_wait);
xbt_replay_action_register("barrier", action_barrier);
xbt_replay_action_register("bcast", action_bcast);
xbt_replay_action_register("reduce", action_reduce);
xbt_replay_action_register("allReduce",action_allReduce);
xbt_replay_action_register("compute", action_compute);
}
xbt_replay_action_runner(*argc, *argv);
}
int MPI_Init(int *argc, char ***argv)
{
int err;
/* call the init routines */
err=PMPI_Init(argc,argv);
/* when this returns, all modules have registered their
needs for extra storage - this number is now fixed */
if (add_status_storage<=0)
{
/* No module requested extra space,
we should really disable this module, but
for now we just ignore it */
add_status_storage=0;
}
return err;
}
int main(int argc, char *argv[]) {
int numproc, rank, len;
char hostname[MPI_MAX_PROCESSOR_NAME];
PMPI_Init(&argc, &argv);
PMPI_Comm_size(MPI_COMM_WORLD, &numproc);
PMPI_Comm_rank(MPI_COMM_WORLD, &rank);
PMPI_Get_processor_name(hostname, &len);
if (rank==0) {
int *freq,i,j;
freq=(int *)malloc(sizeof(int)*numproc);
char *temp;
temp=(char*)malloc(sizeof(char)*(numproc-1));
MPI_Status *stat, *stat1;
stat = (MPI_Status*)malloc(sizeof(MPI_Status)*(numproc-1));
stat1 = (MPI_Status*)malloc(sizeof(MPI_Status)*(numproc-1));
MPI_Request *req;
req = (MPI_Request *)malloc(sizeof(MPI_Request)*(numproc-1));
int N=numproc*numproc;
for(i=1; i<numproc; i++) {
PMPI_Recv(temp+i-1, 1, MPI_CHAR, i, 0, MPI_COMM_WORLD, stat+(i-1));//, req+(i-1)*2);
}
for(i=1; i<numproc; i++) {
PMPI_Recv(freq+i*numproc, numproc, MPI_INT, i, 1, MPI_COMM_WORLD,
stat1+(i-1));
}
printf("echo\n");
// MPI_Waitall((numproc-1), req, stat);
for (i=1; i<numproc; i++) {
printf("Rank %d ", i);
for (j=0; j<numproc; j++) {
if(j!=i) {
int loc = i*numproc+j;
printf("%d ",freq[loc]);
}
}
printf("\n");
}
}
else {
int i, *nsend;
char *rMsg, msg='x';
rMsg=(char*)malloc(sizeof(char));
nsend=(int*)malloc(sizeof(int)*numproc);
// msg=(char*)malloc(sizeof(char));
// memset(msg, 'z', sizeof(char));
memset(nsend, 0, sizeof(int)*numproc);
MPI_Request *req;
req = (MPI_Request *)malloc(sizeof(MPI_Request)*(numproc));
MPI_Status *stat;
stat = (MPI_Status*)malloc(sizeof(MPI_Status)*(numproc-1));
for (i=0; i<numproc; i++) {
if(i!=rank) {
*(nsend+i)+=*(nsend+i)+1;
PMPI_Isend(&msg, 1, MPI_CHAR, i, 0, MPI_COMM_WORLD, &(req[i]));
}
}
// printf("Echo-1\n");
for (i=1; i<numproc; i++) {
if (i!=rank)
PMPI_Recv(rMsg, 1, MPI_CHAR, i, 0, MPI_COMM_WORLD, stat+i-1);
}
// printf("Echo-2\n");
MPI_Isend(nsend, numproc, MPI_INT, 0, 1, MPI_COMM_WORLD, req+numproc);
// MPI_Isend(msg, 1, MPI_CHAR, i, 0, MPI_COMM_WORLD, req+numproc);
// printf("Echo-3\n");
}
PMPI_Finalize();
return(0);
}
int main(int argc, char* argv[])
{
int result = SMPD_SUCCESS;
smpd_host_node_t *host_node_ptr;
smpd_launch_node_t *launch_node_ptr;
smpd_context_t *context;
SMPDU_Sock_set_t set;
SMPDU_Sock_t sock = SMPDU_SOCK_INVALID_SOCK;
smpd_state_t state;
smpd_enter_fn("main");
/* catch an empty command line */
if (argc < 2)
{
mp_print_options();
exit(0);
}
smpd_process.mpiexec_argv0 = argv[0];
/* initialize */
/* FIXME: Get rid of this hack - we already create
* local KVS for all singleton clients by default
*/
putenv("PMI_SMPD_FD=0");
result = PMPI_Init(&argc, &argv);
/* SMPD_CS_ENTER(); */
if (result != SMPD_SUCCESS)
{
smpd_err_printf("SMPD_Init failed,\nerror: %d\n", result);
smpd_exit_fn("main");
return result;
}
result = SMPDU_Sock_init();
if (result != SMPD_SUCCESS)
{
smpd_err_printf("SMPDU_Sock_init failed,\nsock error: %s\n",
get_sock_error_string(result));
smpd_exit_fn("main");
return result;
}
result = smpd_init_process();
if (result != SMPD_SUCCESS)
{
smpd_err_printf("smpd_init_process failed.\n");
goto quit_job;
}
smpd_process.dbg_state = SMPD_DBG_STATE_ERROUT;
/* parse the command line */
smpd_dbg_printf("parsing the command line.\n");
result = mp_parse_command_args(&argc, &argv);
if (result != SMPD_SUCCESS)
{
smpd_err_printf("Unable to parse the mpiexec command arguments.\n");
goto quit_job;
}
/* If we are using MS HPC job scheduler we only connect
* to the local SMPD
*/
if(smpd_process.use_ms_hpc){
char host[100];
int id;
/* Free the current host list */
result = smpd_free_host_list();
if(result != SMPD_SUCCESS){
smpd_err_printf("Unable to free the global host list\n");
goto quit_job;
}
/* Add local host to the host list */
result = smpd_get_hostname(host, 100);
if(result != SMPD_SUCCESS){
smpd_err_printf("Unable to get the local hostname\n");
goto quit_job;
}
result = smpd_get_host_id(host, &id);
if(result != SMPD_SUCCESS){
smpd_err_printf("Unable to get host id for local host\n");
goto quit_job;
}
/* Set the number of PMI procs since they are not launched by mpiexec */
smpd_process.nproc = smpd_process.launch_list->nproc;
smpd_dbg_printf("Adding (%s:%d) == (localhost) to the host list\n", host, id);
}
/* print and see what we've got */
/* debugging output *************/
smpd_dbg_printf("host tree:\n");
host_node_ptr = smpd_process.host_list;
if (!host_node_ptr)
smpd_dbg_printf("<none>\n");
while (host_node_ptr)
{
smpd_dbg_printf(" host: %s, parent: %d, id: %d\n",
host_node_ptr->host,
host_node_ptr->parent, host_node_ptr->id);
host_node_ptr = host_node_ptr->next;
}
smpd_dbg_printf("launch nodes:\n");
launch_node_ptr = smpd_process.launch_list;
if (!launch_node_ptr)
smpd_dbg_printf("<none>\n");
while (launch_node_ptr)
{
smpd_dbg_printf(" iproc: %d, id: %d, exe: %s\n",
launch_node_ptr->iproc, launch_node_ptr->host_id,
launch_node_ptr->exe);
launch_node_ptr = launch_node_ptr->next;
}
/* end debug output *************/
/* set the id of the mpiexec node to zero */
smpd_process.id = 0;
result = SMPDU_Sock_create_set(&set);
if (result != SMPD_SUCCESS)
{
smpd_err_printf("SMPDU_Sock_create_set failed,\nsock error: %s\n", get_sock_error_string(result));
goto quit_job;
}
smpd_process.set = set;
/* Check to see if the user wants to use a remote shell mechanism for launching the processes
* instead of using the smpd process managers.
*/
if (smpd_process.rsh_mpiexec == SMPD_TRUE)
{
/* Do rsh or localonly stuff */
result = mpiexec_rsh();
/* skip over the non-rsh code and go to the cleanup section */
goto quit_job;
}
/* Start the timeout mechanism if specified */
/* This code occurs after the rsh_mpiexec option check because the rsh code implementes timeouts differently */
if (smpd_process.timeout > 0)
{
#ifdef HAVE_WINDOWS_H
/* create a Windows thread to sleep until the timeout expires */
if (smpd_process.timeout_thread == NULL)
{
smpd_process.timeout_thread = CreateThread(NULL, 0, (LPTHREAD_START_ROUTINE)timeout_thread, NULL, 0, NULL);
if (smpd_process.timeout_thread == NULL)
{
printf("Error: unable to create a timeout thread, errno %d.\n", GetLastError());
smpd_exit_fn("mp_parse_command_args");
return SMPD_FAIL;
}
}
#elif defined(SIGALRM)
/* create an alarm to signal mpiexec when the timeout expires */
smpd_signal(SIGALRM, timeout_function);
alarm(smpd_process.timeout);
#elif defined(HAVE_PTHREAD_H)
/* create a pthread to sleep until the timeout expires */
result = pthread_create(&smpd_process.timeout_thread, NULL, timeout_thread, NULL);
if (result != 0)
{
printf("Error: unable to create a timeout thread, errno %d.\n", result);
smpd_exit_fn("mp_parse_command_args");
return SMPD_FAIL;
}
#else
/* no timeout mechanism available */
#endif
}
/* make sure we have a passphrase to authenticate connections to the smpds */
if (smpd_process.passphrase[0] == '\0')
smpd_get_smpd_data("phrase", smpd_process.passphrase, SMPD_PASSPHRASE_MAX_LENGTH);
if (smpd_process.passphrase[0] == '\0')
{
if (smpd_process.noprompt)
{
printf("Error: No smpd passphrase specified through the registry or .smpd file, exiting.\n");
result = SMPD_FAIL;
goto quit_job;
}
printf("Please specify an authentication passphrase for smpd: ");
fflush(stdout);
smpd_get_password(smpd_process.passphrase);
}
/* set the state to create a console session or a job session */
state = smpd_process.do_console ? SMPD_MPIEXEC_CONNECTING_SMPD : SMPD_MPIEXEC_CONNECTING_TREE;
result = smpd_create_context(SMPD_CONTEXT_LEFT_CHILD, set, sock, 1, &context);
if (result != SMPD_SUCCESS)
{
smpd_err_printf("unable to create a context for the first host in the tree.\n");
goto quit_job;
}
#ifdef HAVE_WINDOWS_H
if (!smpd_process.local_root)
{
#endif
/* start connecting the tree by posting a connect to the first host */
result = SMPDU_Sock_post_connect(set, context, smpd_process.host_list->host, smpd_process.port, &sock);
if (result != SMPD_SUCCESS)
{
smpd_err_printf("Unable to connect to '%s:%d',\nsock error: %s\n",
smpd_process.host_list->host, smpd_process.port, get_sock_error_string(result));
goto quit_job;
}
#ifdef HAVE_WINDOWS_H
}
#endif
context->sock = sock;
context->state = state;
context->connect_to = smpd_process.host_list;
#ifdef HAVE_WINDOWS_H
if (smpd_process.local_root)
{
int port;
smpd_context_t *rc_context;
/* The local_root option is implemented by having mpiexec act as the smpd
* and launch the smpd manager. Then mpiexec connects to this manager just
* as if it had been created by a real smpd. This causes all the processes
* destined for the first smpd host to be launched by this child process of
* mpiexec and not the smpd service. This allows for these processes to
* create windows that are visible to the interactive user. It also means
* that the job cannot be run in the context of a user other than the user
* running mpiexec. */
/* get the path to smpd.exe because pszExe is currently mpiexec.exe */
smpd_get_smpd_data("binary", smpd_process.pszExe, SMPD_MAX_EXE_LENGTH);
/* launch the manager process */
result = smpd_start_win_mgr(context, SMPD_FALSE);
if (result != SMPD_SUCCESS)
{
smpd_err_printf("unable to start the local smpd manager.\n");
goto quit_job;
}
/* connect to the manager */
smpd_dbg_printf("connecting a new socket.\n");
port = atol(context->port_str);
if (port < 1)
{
smpd_err_printf("Invalid reconnect port read: %d\n", port);
goto quit_job;
}
result = smpd_create_context(context->type, context->set, SMPDU_SOCK_INVALID_SOCK, context->id, &rc_context);
if (result != SMPD_SUCCESS)
{
smpd_err_printf("unable to create a new context for the reconnection.\n");
goto quit_job;
}
rc_context->state = context->state;
rc_context->write_state = SMPD_RECONNECTING;
context->state = SMPD_CLOSING;
rc_context->connect_to = context->connect_to;
rc_context->connect_return_id = context->connect_return_id;
rc_context->connect_return_tag = context->connect_return_tag;
strcpy(rc_context->host, context->host);
smpd_process.left_context = rc_context;
smpd_dbg_printf("posting a re-connect to %s:%d in %s context.\n", rc_context->connect_to->host, port, smpd_get_context_str(rc_context));
result = SMPDU_Sock_post_connect(rc_context->set, rc_context, rc_context->connect_to->host, port, &rc_context->sock);
if (result != SMPD_SUCCESS)
{
smpd_err_printf("Unable to post a connect to '%s:%d',\nsock error: %s\n",
rc_context->connect_to->host, port, get_sock_error_string(result));
if (smpd_post_abort_command("Unable to connect to '%s:%d',\nsock error: %s\n",
rc_context->connect_to->host, port, get_sock_error_string(result)) != SMPD_SUCCESS)
{
goto quit_job;
}
}
}
else
{
#endif
smpd_process.left_context = context;
result = SMPDU_Sock_set_user_ptr(sock, context);
if (result != SMPD_SUCCESS)
{
smpd_err_printf("unable to set the smpd sock user pointer,\nsock error: %s\n",
get_sock_error_string(result));
goto quit_job;
}
#ifdef HAVE_WINDOWS_H
}
#endif
#ifdef HAVE_WINDOWS_H
{
/* Create a break handler and a socket to handle aborting the job when mpiexec receives break signals */
smpd_context_t *reader_context;
SMPDU_Sock_t sock_reader;
SMPDU_SOCK_NATIVE_FD reader, writer;
smpd_make_socket_loop((SOCKET*)&reader, (SOCKET*)&writer);
result = SMPDU_Sock_native_to_sock(set, reader, NULL, &sock_reader);
result = SMPDU_Sock_native_to_sock(set, writer, NULL, &smpd_process.mpiexec_abort_sock);
result = smpd_create_context(SMPD_CONTEXT_MPIEXEC_ABORT, set, sock_reader, -1, &reader_context);
reader_context->read_state = SMPD_READING_MPIEXEC_ABORT;
result = SMPDU_Sock_post_read(sock_reader, &reader_context->read_cmd.cmd, 1, 1, NULL);
if (!SetConsoleCtrlHandler(mpiexec_ctrl_handler, TRUE))
{
/* Don't error out; allow the job to run without a ctrl handler? */
result = GetLastError();
smpd_dbg_printf("unable to set a ctrl handler for mpiexec, error %d\n", result);
}
}
#endif
result = smpd_enter_at_state(set, state);
if (result != SMPD_SUCCESS)
{
smpd_err_printf("state machine failed.\n");
goto quit_job;
}
quit_job:
if ((result != SMPD_SUCCESS) && (smpd_process.mpiexec_exit_code == 0))
{
smpd_process.mpiexec_exit_code = -1;
}
/* finalize */
smpd_dbg_printf("calling SMPDU_Sock_finalize\n");
result = SMPDU_Sock_finalize();
if (result != SMPD_SUCCESS)
{
smpd_err_printf("SMPDU_Sock_finalize failed,\nsock error: %s\n", get_sock_error_string(result));
}
/* SMPD_Finalize called in smpd_exit()
smpd_dbg_printf("calling SMPD_Finalize\n");
result = PMPI_Finalize();
if (result != SMPD_SUCCESS)
{
smpd_err_printf("SMPD_Finalize failed,\nerror: %d\n", result);
}
*/
#ifdef HAVE_WINDOWS_H
if (smpd_process.hCloseStdinThreadEvent)
SetEvent(smpd_process.hCloseStdinThreadEvent);
if (smpd_process.hStdinThread != NULL)
{
/* close stdin so the input thread will exit */
CloseHandle(GetStdHandle(STD_INPUT_HANDLE));
if (WaitForSingleObject(smpd_process.hStdinThread, 3000) != WAIT_OBJECT_0)
{
TerminateThread(smpd_process.hStdinThread, 321);
}
CloseHandle(smpd_process.hStdinThread);
}
if (smpd_process.hCloseStdinThreadEvent)
{
CloseHandle(smpd_process.hCloseStdinThreadEvent);
smpd_process.hCloseStdinThreadEvent = NULL;
}
#elif defined(USE_PTHREAD_STDIN_REDIRECTION)
smpd_cancel_stdin_thread();
#endif
smpd_exit_fn("main");
/* SMPD_CS_EXIT(); */
return smpd_exit(smpd_process.mpiexec_exit_code);
}
int MPI_Init_thread(int *argc, char ***argv, int required, int *provided)
{
int mpi_errno = MPI_SUCCESS;
int i, j;
int local_rank, local_nprocs, rank, nprocs, user_rank, user_nprocs;
int local_user_rank = -1, local_user_nprocs = -1;
int *tmp_gather_buf = NULL, node_id = 0;
int tmp_bcast_buf[2];
int *ranks_in_user_world = NULL, *ranks_in_world = NULL;
MTCORE_DBG_PRINT_FCNAME();
if (required == 0 && provided == NULL) {
/* default init */
mpi_errno = PMPI_Init(argc, argv);
if (mpi_errno != MPI_SUCCESS)
goto fn_fail;
}
else {
/* user init thread */
mpi_errno = PMPI_Init_thread(argc, argv, required, provided);
if (mpi_errno != MPI_SUCCESS)
goto fn_fail;
}
PMPI_Comm_size(MPI_COMM_WORLD, &nprocs);
PMPI_Comm_rank(MPI_COMM_WORLD, &rank);
MTCORE_MY_RANK_IN_WORLD = rank;
mpi_errno = MTCORE_Initialize_env();
if (mpi_errno != MPI_SUCCESS)
goto fn_fail;
/* Get a communicator only containing processes with shared memory */
mpi_errno = PMPI_Comm_split_type(MPI_COMM_WORLD, MPI_COMM_TYPE_SHARED, 0,
MPI_INFO_NULL, &MTCORE_COMM_LOCAL);
if (mpi_errno != MPI_SUCCESS)
goto fn_fail;
/* Check number of helpers and number of processes */
PMPI_Comm_rank(MTCORE_COMM_LOCAL, &local_rank);
PMPI_Comm_size(MTCORE_COMM_LOCAL, &local_nprocs);
if (local_nprocs < 2) {
fprintf(stderr, "No user process found, please run with more than 2 process per node\n");
mpi_errno = -1;
goto fn_fail;
}
if (MTCORE_ENV.num_h < 1 || MTCORE_ENV.num_h >= local_nprocs) {
fprintf(stderr, "Wrong value of number of helpers, %d. lt 1 or ge %d.\n",
MTCORE_ENV.num_h, local_nprocs);
mpi_errno = -1;
goto fn_fail;
}
/* Specify the first N local processes to be Helper processes */
MTCORE_H_RANKS_IN_LOCAL = calloc(MTCORE_ENV.num_h, sizeof(int));
MTCORE_H_RANKS_IN_WORLD = calloc(MTCORE_ENV.num_h, sizeof(int));
for (i = 0; i < MTCORE_ENV.num_h; i++) {
MTCORE_H_RANKS_IN_LOCAL[i] = i;
}
mpi_errno = PMPI_Comm_group(MPI_COMM_WORLD, &MTCORE_GROUP_WORLD);
mpi_errno = PMPI_Comm_group(MTCORE_COMM_LOCAL, &MTCORE_GROUP_LOCAL);
mpi_errno = PMPI_Group_translate_ranks(MTCORE_GROUP_LOCAL, MTCORE_ENV.num_h,
MTCORE_H_RANKS_IN_LOCAL, MTCORE_GROUP_WORLD,
MTCORE_H_RANKS_IN_WORLD);
if (mpi_errno != MPI_SUCCESS)
goto fn_fail;
/* Create a user comm_world including all the users,
* user will access it instead of comm_world */
mpi_errno = PMPI_Comm_split(MPI_COMM_WORLD,
local_rank < MTCORE_ENV.num_h, 0, &MTCORE_COMM_USER_WORLD);
if (mpi_errno != MPI_SUCCESS)
goto fn_fail;
PMPI_Comm_size(MTCORE_COMM_USER_WORLD, &user_nprocs);
PMPI_Comm_rank(MTCORE_COMM_USER_WORLD, &user_rank);
PMPI_Comm_group(MTCORE_COMM_USER_WORLD, &MTCORE_GROUP_USER_WORLD);
/* Create a user comm_local */
mpi_errno = PMPI_Comm_split(MTCORE_COMM_LOCAL,
local_rank < MTCORE_ENV.num_h, 0, &MTCORE_COMM_USER_LOCAL);
if (mpi_errno != MPI_SUCCESS)
goto fn_fail;
/* Create a helper comm_local */
mpi_errno = PMPI_Comm_split(MTCORE_COMM_LOCAL,
local_rank < MTCORE_ENV.num_h, 1, &MTCORE_COMM_HELPER_LOCAL);
if (mpi_errno != MPI_SUCCESS)
goto fn_fail;
/* Exchange node id among local processes */
/* -Only users create a user root communicator for exchanging local informations
* between different nodes*/
if (local_rank >= MTCORE_ENV.num_h) {
PMPI_Comm_rank(MTCORE_COMM_USER_LOCAL, &local_user_rank);
PMPI_Comm_size(MTCORE_COMM_USER_LOCAL, &local_user_nprocs);
mpi_errno = PMPI_Comm_split(MTCORE_COMM_USER_WORLD,
local_user_rank == 0, 1, &MTCORE_COMM_UR_WORLD);
if (mpi_errno != MPI_SUCCESS)
goto fn_fail;
/* -Only user roots determine a node id for each USER processes */
if (local_user_rank == 0) {
PMPI_Comm_size(MTCORE_COMM_UR_WORLD, &MTCORE_NUM_NODES);
PMPI_Comm_rank(MTCORE_COMM_UR_WORLD, &MTCORE_MY_NODE_ID);
tmp_bcast_buf[0] = MTCORE_MY_NODE_ID;
tmp_bcast_buf[1] = MTCORE_NUM_NODES;
}
}
/* -User root broadcasts to other local processes */
PMPI_Bcast(tmp_bcast_buf, 2, MPI_INT, MTCORE_ENV.num_h, MTCORE_COMM_LOCAL);
MTCORE_MY_NODE_ID = tmp_bcast_buf[0];
MTCORE_NUM_NODES = tmp_bcast_buf[1];
/* Exchange node id and Helper ranks among world processes */
ranks_in_world = calloc(nprocs, sizeof(int));
ranks_in_user_world = calloc(nprocs, sizeof(int));
for (i = 0; i < nprocs; i++) {
ranks_in_world[i] = i;
}
mpi_errno = PMPI_Group_translate_ranks(MTCORE_GROUP_WORLD, nprocs,
ranks_in_world, MTCORE_GROUP_USER_WORLD,
ranks_in_user_world);
if (mpi_errno != MPI_SUCCESS)
goto fn_fail;
MTCORE_ALL_NODE_IDS = calloc(nprocs, sizeof(int));
MTCORE_ALL_H_RANKS_IN_WORLD = calloc(user_nprocs * MTCORE_ENV.num_h, sizeof(int));
MTCORE_ALL_UNIQUE_H_RANKS_IN_WORLD = calloc(MTCORE_NUM_NODES * MTCORE_ENV.num_h, sizeof(int));
tmp_gather_buf = calloc(nprocs * (1 + MTCORE_ENV.num_h), sizeof(int));
tmp_gather_buf[rank * (1 + MTCORE_ENV.num_h)] = MTCORE_MY_NODE_ID;
for (i = 0; i < MTCORE_ENV.num_h; i++) {
tmp_gather_buf[rank * (1 + MTCORE_ENV.num_h) + i + 1] = MTCORE_H_RANKS_IN_WORLD[i];
}
mpi_errno = PMPI_Allgather(MPI_IN_PLACE, 0, MPI_DATATYPE_NULL,
tmp_gather_buf, 1 + MTCORE_ENV.num_h, MPI_INT, MPI_COMM_WORLD);
if (mpi_errno != MPI_SUCCESS)
goto fn_fail;
for (i = 0; i < nprocs; i++) {
int i_user_rank = 0;
node_id = tmp_gather_buf[i * (1 + MTCORE_ENV.num_h)];
MTCORE_ALL_NODE_IDS[i] = node_id;
/* Only copy helper ranks for user processes */
i_user_rank = ranks_in_user_world[i];
if (i_user_rank != MPI_UNDEFINED) {
for (j = 0; j < MTCORE_ENV.num_h; j++) {
MTCORE_ALL_H_RANKS_IN_WORLD[i_user_rank * MTCORE_ENV.num_h + j] =
tmp_gather_buf[i * (1 + MTCORE_ENV.num_h) + j + 1];
MTCORE_ALL_UNIQUE_H_RANKS_IN_WORLD[node_id * MTCORE_ENV.num_h + j] =
tmp_gather_buf[i * (1 + MTCORE_ENV.num_h) + j + 1];
}
}
}
#ifdef DEBUG
MTCORE_DBG_PRINT("Debug gathered info ***** \n");
for (i = 0; i < nprocs; i++) {
MTCORE_DBG_PRINT("node_id[%d]: %d\n", i, MTCORE_ALL_NODE_IDS[i]);
}
#endif
/* USER processes */
if (local_rank >= MTCORE_ENV.num_h) {
/* Get user ranks in world */
for (i = 0; i < user_nprocs; i++)
ranks_in_user_world[i] = i;
MTCORE_USER_RANKS_IN_WORLD = calloc(user_nprocs, sizeof(int));
mpi_errno = PMPI_Group_translate_ranks(MTCORE_GROUP_USER_WORLD, user_nprocs,
ranks_in_user_world, MTCORE_GROUP_WORLD,
MTCORE_USER_RANKS_IN_WORLD);
if (mpi_errno != MPI_SUCCESS)
goto fn_fail;
#ifdef DEBUG
for (i = 0; i < user_nprocs; i++) {
MTCORE_DBG_PRINT("helper_rank_in_world[%d]:\n", i);
for (j = 0; j < MTCORE_ENV.num_h; j++) {
MTCORE_DBG_PRINT(" %d\n", MTCORE_ALL_H_RANKS_IN_WORLD[i * MTCORE_ENV.num_h + j]);
}
}
#endif
MTCORE_DBG_PRINT("I am user, %d/%d in world, %d/%d in local, %d/%d in user world, "
"%d/%d in user local, node_id %d\n", rank, nprocs, local_rank,
local_nprocs, user_rank, user_nprocs, local_user_rank,
local_user_nprocs, MTCORE_MY_NODE_ID);
MTCORE_Init_win_cache();
}
/* Helper processes */
/* TODO: Helper process should not run user program */
else {
/* free local buffers before enter helper main function */
if (tmp_gather_buf)
free(tmp_gather_buf);
if (ranks_in_user_world)
free(ranks_in_user_world);
if (ranks_in_world)
free(ranks_in_world);
MTCORE_DBG_PRINT("I am helper, %d/%d in world, %d/%d in local, node_id %d\n", rank,
nprocs, local_rank, local_nprocs, MTCORE_MY_NODE_ID);
run_h_main();
exit(0);
}
fn_exit:
if (tmp_gather_buf)
free(tmp_gather_buf);
if (ranks_in_user_world)
free(ranks_in_user_world);
if (ranks_in_world)
free(ranks_in_world);
return mpi_errno;
fn_fail:
/* --BEGIN ERROR HANDLING-- */
if (MTCORE_COMM_USER_WORLD != MPI_COMM_NULL) {
MTCORE_DBG_PRINT("free MTCORE_COMM_USER_WORLD\n");
PMPI_Comm_free(&MTCORE_COMM_USER_WORLD);
}
if (MTCORE_COMM_LOCAL != MPI_COMM_NULL) {
MTCORE_DBG_PRINT("free MTCORE_COMM_LOCAL\n");
PMPI_Comm_free(&MTCORE_COMM_LOCAL);
}
if (MTCORE_COMM_USER_LOCAL != MPI_COMM_NULL) {
MTCORE_DBG_PRINT("free MTCORE_COMM_USER_LOCAL\n");
PMPI_Comm_free(&MTCORE_COMM_USER_LOCAL);
}
if (MTCORE_COMM_UR_WORLD != MPI_COMM_NULL) {
MTCORE_DBG_PRINT("free MTCORE_COMM_UR_WORLD\n");
PMPI_Comm_free(&MTCORE_COMM_UR_WORLD);
}
if (MTCORE_COMM_HELPER_LOCAL != MPI_COMM_NULL) {
MTCORE_DBG_PRINT("free MTCORE_COMM_HELPER_LOCAL\n");
PMPI_Comm_free(&MTCORE_COMM_HELPER_LOCAL);
}
if (MTCORE_GROUP_WORLD != MPI_GROUP_NULL)
PMPI_Group_free(&MTCORE_GROUP_WORLD);
if (MTCORE_GROUP_LOCAL != MPI_GROUP_NULL)
PMPI_Group_free(&MTCORE_GROUP_LOCAL);
if (MTCORE_GROUP_USER_WORLD != MPI_GROUP_NULL)
PMPI_Group_free(&MTCORE_GROUP_USER_WORLD);
if (MTCORE_H_RANKS_IN_WORLD)
free(MTCORE_H_RANKS_IN_WORLD);
if (MTCORE_H_RANKS_IN_LOCAL)
free(MTCORE_H_RANKS_IN_LOCAL);
if (MTCORE_ALL_H_RANKS_IN_WORLD)
free(MTCORE_ALL_H_RANKS_IN_WORLD);
if (MTCORE_ALL_UNIQUE_H_RANKS_IN_WORLD)
free(MTCORE_ALL_UNIQUE_H_RANKS_IN_WORLD);
if (MTCORE_ALL_NODE_IDS)
free(MTCORE_ALL_NODE_IDS);
if (MTCORE_USER_RANKS_IN_WORLD)
free(MTCORE_USER_RANKS_IN_WORLD);
MTCORE_Destroy_win_cache();
/* Reset global variables */
MTCORE_COMM_USER_WORLD = MPI_COMM_NULL;
MTCORE_COMM_USER_LOCAL = MPI_COMM_NULL;
MTCORE_COMM_LOCAL = MPI_COMM_NULL;
MTCORE_ALL_H_RANKS_IN_WORLD = NULL;
MTCORE_ALL_NODE_IDS = NULL;
PMPI_Abort(MPI_COMM_WORLD, 0);
goto fn_exit;
/* --END ERROR HANDLING-- */
}
int MPI_Init(int *argc, char ***argv)
{
return PMPI_Init(argc, argv);
}
static int PNMPI_Common_MPI_Init(int * _pnmpi_arg_0, char * * * _pnmpi_arg_1)
{
int returnVal;
pnmpi_PreInit(); /* this will never fail */
if (NOT_ACTIVATED(MPI_Init_MAJOR,MPI_Init_MINOR))
{
#ifdef COMPILE_FOR_FORTRAN
if (init_was_fortran)
pmpi_init_(&returnVal);
else
#endif
returnVal=PMPI_Init(_pnmpi_arg_0, _pnmpi_arg_1);
}
else
returnVal=Internal_XMPI_Init(_pnmpi_arg_0, _pnmpi_arg_1);
if (returnVal!=MPI_SUCCESS)
return returnVal;
DBGLATEINIT();
STATUSINIT();
DBGPRINT1("Leaving Init");
STATUSPRINT1("");
STATUSPRINT1("\t\t ---------------------------");
STATUSPRINT1("\t\t | P^N-MPI Interface |");
STATUSPRINT1("\t\t | Martin Schulz, 2005, LLNL |");
STATUSPRINT1("\t\t ---------------------------");
STATUSPRINT1("");
{
int i;
module_servlist_p serv;
module_globlist_p glob;
module_arg_t *args;
STATUSPRINT1("Number of modules: %i",modules.num);
STATUSPRINT1("Pcontrol Setting: %i",modules.pcontrol);
STATUSPRINT1("");
for (i=0; i<modules.num; i++)
{
if (modules.module[i]->registered)
{
STATUSPRINT1("Module %s: registered as %s (Pctrl %i)",
modules.module[i]->name,
modules.module[i]->username,
modules.module[i]->pcontrol);
}
else
{
if (modules.module[i]->stack_delimiter)
{ STATUSPRINT1("Stack %s: not registered",
modules.module[i]->name);
}
else
{ STATUSPRINT1("Module %s: not registered (Pctrl %i)",
modules.module[i]->name,
modules.module[i]->pcontrol);
}
}
for (args=modules.module[i]->args; args!=NULL; args=args->next)
{
STATUSPRINT1("\tArgument: %s = %s",
args->name,args->value);
}
for (serv=modules.module[i]->services; serv!=NULL; serv=serv->next)
{
STATUSPRINT1("\tService: %s (%s)",
serv->desc.name,serv->desc.sig);
}
for (glob=modules.module[i]->globals; glob!=NULL; glob=glob->next)
{
STATUSPRINT1("\tGlobal: %s (%c)",
glob->desc.name,glob->desc.sig);
}
}
STATUSPRINT1("");
}
return returnVal;
}
int main(int argc, char **argv)
{
int *buf, i, rank, nints, len;
char *filename, *tmp;
int errs=0, toterrs;
MPI_File fh;
MPI_Status status;
PMPI_Init(&argc,&argv);
PMPI_Comm_rank(MPI_COMM_WORLD, &rank);
/* process 0 takes the file name as a command-line argument and
broadcasts it to other processes */
if (!rank) {
i = 1;
while ((i < argc) && strcmp("-fname", *argv)) {
i++;
argv++;
}
if (i >= argc) {
fprintf(stderr, "\n*# Usage: simple -fname filename\n\n");
PMPI_Abort(MPI_COMM_WORLD, 1);
}
argv++;
len = strlen(*argv);
filename = (char *) malloc(len+10);
strcpy(filename, *argv);
PMPI_Bcast(&len, 1, MPI_INT, 0, MPI_COMM_WORLD);
PMPI_Bcast(filename, len+10, MPI_CHAR, 0, MPI_COMM_WORLD);
}
else {
PMPI_Bcast(&len, 1, MPI_INT, 0, MPI_COMM_WORLD);
filename = (char *) malloc(len+10);
PMPI_Bcast(filename, len+10, MPI_CHAR, 0, MPI_COMM_WORLD);
}
buf = (int *) malloc(SIZE);
nints = SIZE/sizeof(int);
for (i=0; i<nints; i++) buf[i] = rank*100000 + i;
/* each process opens a separate file called filename.'myrank' */
tmp = (char *) malloc(len+10);
strcpy(tmp, filename);
sprintf(filename, "%s.%d", tmp, rank);
PMPI_File_open(MPI_COMM_SELF, filename, MPI_MODE_CREATE | MPI_MODE_RDWR,
MPI_INFO_NULL, &fh);
PMPI_File_write(fh, buf, nints, MPI_INT, &status);
PMPI_File_close(&fh);
/* reopen the file and read the data back */
for (i=0; i<nints; i++) buf[i] = 0;
PMPI_File_open(MPI_COMM_SELF, filename, MPI_MODE_CREATE | MPI_MODE_RDWR,
MPI_INFO_NULL, &fh);
PMPI_File_read(fh, buf, nints, MPI_INT, &status);
PMPI_File_close(&fh);
/* check if the data read is correct */
for (i=0; i<nints; i++) {
if (buf[i] != (rank*100000 + i)) {
errs++;
fprintf(stderr, "Process %d: error, read %d, should be %d\n", rank, buf[i], rank*100000+i);
}
}
MPI_Allreduce( &errs, &toterrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD );
if (rank == 0) {
if( toterrs > 0) {
fprintf( stderr, "Found %d errors\n", toterrs );
}
else {
fprintf( stdout, " No Errors\n" );
}
}
free(buf);
free(filename);
free(tmp);
PMPI_Finalize();
return 0;
}
