int MPI_Init(int* argc, char*** argv)
{
int result, MPIT_result;
int provided;
result = PMPI_Init(argc, argv);
PMPI_Comm_size(MPI_COMM_WORLD, &comm_world_size);
PMPI_Comm_rank(MPI_COMM_WORLD, &comm_world_rank);
MPIT_result = MPI_T_init_thread(MPI_THREAD_SINGLE, &provided);
if (MPIT_result != MPI_SUCCESS) {
fprintf(stderr, "ERROR : failed to intialize MPI_T interface, preventing to get monitoring results: check your OpenMPI installation\n");
PMPI_Abort(MPI_COMM_WORLD, MPIT_result);
}
MPIT_result = MPI_T_pvar_session_create(&session);
if (MPIT_result != MPI_SUCCESS) {
fprintf(stderr, "ERROR : failed to create MPI_T session, preventing to get monitoring results: check your OpenMPI installation\n");
PMPI_Abort(MPI_COMM_WORLD, MPIT_result);
}
init_monitoring_result("pml_monitoring_messages_count", &counts);
init_monitoring_result("pml_monitoring_messages_size", &sizes);
start_monitoring_result(&counts);
start_monitoring_result(&sizes);
return result;
}
int MPI_Init_thread(int* argc, char*** argv, int required, int* provided)
{
parse_dynamic_server(getenv("EPLIB_DYNAMIC_SERVER"));
/* Initialize MPI */
/* Special handling for async thread */
int ret;
if (dynamic_server == DYNAMIC_SERVER_ASYNCTHREAD)
{
ret = PMPI_Init_thread(argc, argv, MPI_THREAD_MULTIPLE, provided);
if (*provided != MPI_THREAD_MULTIPLE)
{
PRINT("Requested thread level not provided.\n");
PMPI_Abort(MPI_COMM_WORLD, -1);
}
}
else
{
ret = PMPI_Init_thread(argc, argv, required, provided);
if (*provided != required)
{
PRINT("Requested thread level not provided.\n");
PMPI_Abort(MPI_COMM_WORLD, -1);
}
}
/* Initialize EPLIB */
EPLIB_init();
return ret;
}
/* --BEGIN DEBUG-- */
void MPITEST_Group_create( int nproc, int myrank, MPI_Group *new_group )
{
MPID_Group *new_group_ptr;
int i;
new_group_ptr = (MPID_Group *)MPIU_Handle_obj_alloc( &MPID_Group_mem );
if (!new_group_ptr) {
fprintf( stderr, "Could not create a new group\n" );
PMPI_Abort( MPI_COMM_WORLD, 1 );
}
MPIU_Object_set_ref( new_group_ptr, 1 );
new_group_ptr->lrank_to_lpid = (MPID_Group_pmap_t *)MPIU_Malloc( nproc * sizeof(MPID_Group_pmap_t) );
if (!new_group_ptr->lrank_to_lpid) {
fprintf( stderr, "Could not create lrank map for new group\n" );
PMPI_Abort( MPI_COMM_WORLD, 1 );
}
new_group_ptr->rank = MPI_UNDEFINED;
for (i=0; i<nproc; i++) {
new_group_ptr->lrank_to_lpid[i].lrank = i;
new_group_ptr->lrank_to_lpid[i].lpid = i;
}
new_group_ptr->size = nproc;
new_group_ptr->rank = myrank;
new_group_ptr->idx_of_first_lpid = -1;
*new_group = new_group_ptr->handle;
}
void init_monitoring_result(const char * pvar_name, monitoring_result * res)
{
int count;
int MPIT_result;
MPI_Comm comm_world = MPI_COMM_WORLD;
res->pvar_name = strdup(pvar_name);
MPIT_result = MPI_T_pvar_get_index(res->pvar_name, MPI_T_PVAR_CLASS_SIZE, &(res->pvar_idx));
if (MPIT_result != MPI_SUCCESS) {
fprintf(stderr, "ERROR : cannot find monitoring MPI_T \"%s\" pvar, check that you have monitoring pml\n", pvar_name);
PMPI_Abort(MPI_COMM_WORLD, MPIT_result);
}
MPIT_result = MPI_T_pvar_handle_alloc(session, res->pvar_idx, comm_world, &(res->pvar_handle), &count);
if (MPIT_result != MPI_SUCCESS) {
fprintf(stderr, "ERROR : failed to allocate handle on \"%s\" pvar, check that you have monitoring pml\n", pvar_name);
PMPI_Abort(MPI_COMM_WORLD, MPIT_result);
}
if (count != comm_world_size) {
fprintf(stderr, "ERROR : COMM_WORLD has %d ranks \"%s\" pvar contains %d values, check that you have monitoring pml\n", comm_world_size, pvar_name, count);
PMPI_Abort(MPI_COMM_WORLD, count);
}
res->vector = (uint64_t *) malloc(comm_world_size * sizeof(uint64_t));
}
void get_monitoring_result(monitoring_result * res)
{
int MPIT_result;
MPIT_result = MPI_T_pvar_read(session, res->pvar_handle, res->vector);
if (MPIT_result != MPI_SUCCESS) {
fprintf(stderr, "ERROR : failed to read \"%s\" pvar, check that you have enabled the monitoring pml\n", res->pvar_name);
PMPI_Abort(MPI_COMM_WORLD, MPIT_result);
}
}
void
mpiPi_abort (char *fmt, ...)
{
va_list args;
FILE *fp = mpiPi.stderr_;
va_start (args, fmt);
fprintf (fp, "\n\n%s: ABORTING: ", mpiPi.toolname);
vfprintf (fp, fmt, args);
va_end (args);
fflush (fp);
PMPI_Abort (mpiPi.comm, -1);
}
void primary_mapp(int rank, int commsize)
{
int my_id;
old_mapp = malloc(sizeof(*old_mapp) * commsize);
new_mapp = malloc(sizeof(*old_mapp) * commsize);
if (new_mapp == NULL || old_mapp == NULL) {
fprintf(stderr, "new_mapping error or old_mapping\n");
PMPI_Abort(MPI_COMM_WORLD, -1);
}
my_id = getnodeid();
if (rank == 0) {
PMPI_Gather(&my_id, 1, MPI_INT, old_mapp, 1,
MPI_INT, 0, MPI_COMM_WORLD);
} else {
PMPI_Gather(&my_id, 1, MPI_INT, NULL, 0, MPI_INT, 0, MPI_COMM_WORLD);
}
}
int main(int argc, char **argv)
{
int *buf, i, rank, nints, len;
char *filename, *tmp;
int errs=0, toterrs;
MPI_File fh;
MPI_Status status;
PMPI_Init(&argc,&argv);
PMPI_Comm_rank(MPI_COMM_WORLD, &rank);
/* process 0 takes the file name as a command-line argument and
broadcasts it to other processes */
if (!rank) {
i = 1;
while ((i < argc) && strcmp("-fname", *argv)) {
i++;
argv++;
}
if (i >= argc) {
fprintf(stderr, "\n*# Usage: simple -fname filename\n\n");
PMPI_Abort(MPI_COMM_WORLD, 1);
}
argv++;
len = strlen(*argv);
filename = (char *) malloc(len+10);
strcpy(filename, *argv);
PMPI_Bcast(&len, 1, MPI_INT, 0, MPI_COMM_WORLD);
PMPI_Bcast(filename, len+10, MPI_CHAR, 0, MPI_COMM_WORLD);
}
else {
PMPI_Bcast(&len, 1, MPI_INT, 0, MPI_COMM_WORLD);
filename = (char *) malloc(len+10);
PMPI_Bcast(filename, len+10, MPI_CHAR, 0, MPI_COMM_WORLD);
}
buf = (int *) malloc(SIZE);
nints = SIZE/sizeof(int);
for (i=0; i<nints; i++) buf[i] = rank*100000 + i;
/* each process opens a separate file called filename.'myrank' */
tmp = (char *) malloc(len+10);
strcpy(tmp, filename);
sprintf(filename, "%s.%d", tmp, rank);
PMPI_File_open(MPI_COMM_SELF, filename, MPI_MODE_CREATE | MPI_MODE_RDWR,
MPI_INFO_NULL, &fh);
PMPI_File_write(fh, buf, nints, MPI_INT, &status);
PMPI_File_close(&fh);
/* reopen the file and read the data back */
for (i=0; i<nints; i++) buf[i] = 0;
PMPI_File_open(MPI_COMM_SELF, filename, MPI_MODE_CREATE | MPI_MODE_RDWR,
MPI_INFO_NULL, &fh);
PMPI_File_read(fh, buf, nints, MPI_INT, &status);
PMPI_File_close(&fh);
/* check if the data read is correct */
for (i=0; i<nints; i++) {
if (buf[i] != (rank*100000 + i)) {
errs++;
fprintf(stderr, "Process %d: error, read %d, should be %d\n", rank, buf[i], rank*100000+i);
}
}
MPI_Allreduce( &errs, &toterrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD );
if (rank == 0) {
if( toterrs > 0) {
fprintf( stderr, "Found %d errors\n", toterrs );
}
else {
fprintf( stdout, " No Errors\n" );
}
}
free(buf);
free(filename);
free(tmp);
PMPI_Finalize();
return 0;
}
int MPI_Abort( MPI_Comm comm, int errorcode )
{
_MPI_COVERAGE();
return PMPI_Abort(comm, errorcode);
}
int MPI_Abort(MPI_Comm comm, int errorcode)
{
return PMPI_Abort(comm, errorcode);
}
int MPID_Comm_failed_bitarray(MPID_Comm *comm_ptr, uint32_t **bitarray, int acked)
{
int mpi_errno = MPI_SUCCESS;
int size, i;
uint32_t bit;
int *failed_procs, *group_procs;
MPID_Group *failed_group, *comm_group;
MPIU_CHKLMEM_DECL(2);
MPIDI_STATE_DECL(MPID_STATE_COMM_FAILED_BITARRAY);
MPIDI_FUNC_ENTER(MPID_STATE_COMM_FAILED_BITARRAY);
/* TODO - Fix this for intercommunicators */
size = comm_ptr->local_size;
/* We can fit sizeof(uint32_t) * 8 ranks in one uint64_t so divide the
* size by that */
/* This buffer will be handed back to the calling function so we use a
* "real" malloc here and expect the caller to free the buffer later. The
* other buffers in this function are temporary and will be automatically
* cleaned up at the end of the function. */
*bitarray = (uint32_t *) MPIU_Malloc(sizeof(uint32_t) * (size / (sizeof(uint32_t) * 8)+1));
if (!(*bitarray)) {
fprintf(stderr, "Could not allocate space for bitarray\n");
PMPI_Abort(MPI_COMM_WORLD, 1);
}
for (i = 0; i <= size/(sizeof(uint32_t)*8); i++) *bitarray[i] = 0;
mpi_errno = MPIDI_CH3U_Check_for_failed_procs();
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
if (acked)
MPIDI_CH3U_Get_failed_group(comm_ptr->dev.last_ack_rank, &failed_group);
else
MPIDI_CH3U_Get_failed_group(-2, &failed_group);
if (failed_group == MPID_Group_empty) goto fn_exit;
MPIU_CHKLMEM_MALLOC(group_procs, int *, sizeof(int)*failed_group->size, mpi_errno, "group_procs");
for (i = 0; i < failed_group->size; i++) group_procs[i] = i;
MPIU_CHKLMEM_MALLOC(failed_procs, int *, sizeof(int)*failed_group->size, mpi_errno, "failed_procs");
MPIR_Comm_group_impl(comm_ptr, &comm_group);
MPIR_Group_translate_ranks_impl(failed_group, failed_group->size, group_procs, comm_group, failed_procs);
/* The bits will actually be ordered in decending order rather than
* ascending. This is purely for readability since it makes no practical
* difference. So if the bits look like this:
*
* 10001100 01001000 00000000 00000001
*
* Then processes 1, 5, 6, 9, 12, and 32 have failed. */
for (i = 0; i < failed_group->size; i++) {
bit = 0x80000000;
bit >>= failed_procs[i] % (sizeof(uint32_t) * 8);
*bitarray[failed_procs[i] / (sizeof(uint32_t) * 8)] |= bit;
}
MPIR_Group_free_impl(comm_group);
fn_exit:
MPIU_CHKLMEM_FREEALL();
MPIDI_FUNC_EXIT(MPID_STATE_COMM_FAILED_BITARRAY);
return mpi_errno;
fn_fail:
goto fn_exit;
}
int MPI_Init_thread(int *argc, char ***argv, int required, int *provided)
{
int mpi_errno = MPI_SUCCESS;
int i, j;
int local_rank, local_nprocs, rank, nprocs, user_rank, user_nprocs;
int local_user_rank = -1, local_user_nprocs = -1;
int *tmp_gather_buf = NULL, node_id = 0;
int tmp_bcast_buf[2];
int *ranks_in_user_world = NULL, *ranks_in_world = NULL;
MTCORE_DBG_PRINT_FCNAME();
if (required == 0 && provided == NULL) {
/* default init */
mpi_errno = PMPI_Init(argc, argv);
if (mpi_errno != MPI_SUCCESS)
goto fn_fail;
}
else {
/* user init thread */
mpi_errno = PMPI_Init_thread(argc, argv, required, provided);
if (mpi_errno != MPI_SUCCESS)
goto fn_fail;
}
PMPI_Comm_size(MPI_COMM_WORLD, &nprocs);
PMPI_Comm_rank(MPI_COMM_WORLD, &rank);
MTCORE_MY_RANK_IN_WORLD = rank;
mpi_errno = MTCORE_Initialize_env();
if (mpi_errno != MPI_SUCCESS)
goto fn_fail;
/* Get a communicator only containing processes with shared memory */
mpi_errno = PMPI_Comm_split_type(MPI_COMM_WORLD, MPI_COMM_TYPE_SHARED, 0,
MPI_INFO_NULL, &MTCORE_COMM_LOCAL);
if (mpi_errno != MPI_SUCCESS)
goto fn_fail;
/* Check number of helpers and number of processes */
PMPI_Comm_rank(MTCORE_COMM_LOCAL, &local_rank);
PMPI_Comm_size(MTCORE_COMM_LOCAL, &local_nprocs);
if (local_nprocs < 2) {
fprintf(stderr, "No user process found, please run with more than 2 process per node\n");
mpi_errno = -1;
goto fn_fail;
}
if (MTCORE_ENV.num_h < 1 || MTCORE_ENV.num_h >= local_nprocs) {
fprintf(stderr, "Wrong value of number of helpers, %d. lt 1 or ge %d.\n",
MTCORE_ENV.num_h, local_nprocs);
mpi_errno = -1;
goto fn_fail;
}
/* Specify the first N local processes to be Helper processes */
MTCORE_H_RANKS_IN_LOCAL = calloc(MTCORE_ENV.num_h, sizeof(int));
MTCORE_H_RANKS_IN_WORLD = calloc(MTCORE_ENV.num_h, sizeof(int));
for (i = 0; i < MTCORE_ENV.num_h; i++) {
MTCORE_H_RANKS_IN_LOCAL[i] = i;
}
mpi_errno = PMPI_Comm_group(MPI_COMM_WORLD, &MTCORE_GROUP_WORLD);
mpi_errno = PMPI_Comm_group(MTCORE_COMM_LOCAL, &MTCORE_GROUP_LOCAL);
mpi_errno = PMPI_Group_translate_ranks(MTCORE_GROUP_LOCAL, MTCORE_ENV.num_h,
MTCORE_H_RANKS_IN_LOCAL, MTCORE_GROUP_WORLD,
MTCORE_H_RANKS_IN_WORLD);
if (mpi_errno != MPI_SUCCESS)
goto fn_fail;
/* Create a user comm_world including all the users,
* user will access it instead of comm_world */
mpi_errno = PMPI_Comm_split(MPI_COMM_WORLD,
local_rank < MTCORE_ENV.num_h, 0, &MTCORE_COMM_USER_WORLD);
if (mpi_errno != MPI_SUCCESS)
goto fn_fail;
PMPI_Comm_size(MTCORE_COMM_USER_WORLD, &user_nprocs);
PMPI_Comm_rank(MTCORE_COMM_USER_WORLD, &user_rank);
PMPI_Comm_group(MTCORE_COMM_USER_WORLD, &MTCORE_GROUP_USER_WORLD);
/* Create a user comm_local */
mpi_errno = PMPI_Comm_split(MTCORE_COMM_LOCAL,
local_rank < MTCORE_ENV.num_h, 0, &MTCORE_COMM_USER_LOCAL);
if (mpi_errno != MPI_SUCCESS)
goto fn_fail;
/* Create a helper comm_local */
mpi_errno = PMPI_Comm_split(MTCORE_COMM_LOCAL,
local_rank < MTCORE_ENV.num_h, 1, &MTCORE_COMM_HELPER_LOCAL);
if (mpi_errno != MPI_SUCCESS)
goto fn_fail;
/* Exchange node id among local processes */
/* -Only users create a user root communicator for exchanging local informations
* between different nodes*/
if (local_rank >= MTCORE_ENV.num_h) {
PMPI_Comm_rank(MTCORE_COMM_USER_LOCAL, &local_user_rank);
PMPI_Comm_size(MTCORE_COMM_USER_LOCAL, &local_user_nprocs);
mpi_errno = PMPI_Comm_split(MTCORE_COMM_USER_WORLD,
local_user_rank == 0, 1, &MTCORE_COMM_UR_WORLD);
if (mpi_errno != MPI_SUCCESS)
goto fn_fail;
/* -Only user roots determine a node id for each USER processes */
if (local_user_rank == 0) {
PMPI_Comm_size(MTCORE_COMM_UR_WORLD, &MTCORE_NUM_NODES);
PMPI_Comm_rank(MTCORE_COMM_UR_WORLD, &MTCORE_MY_NODE_ID);
tmp_bcast_buf[0] = MTCORE_MY_NODE_ID;
tmp_bcast_buf[1] = MTCORE_NUM_NODES;
}
}
/* -User root broadcasts to other local processes */
PMPI_Bcast(tmp_bcast_buf, 2, MPI_INT, MTCORE_ENV.num_h, MTCORE_COMM_LOCAL);
MTCORE_MY_NODE_ID = tmp_bcast_buf[0];
MTCORE_NUM_NODES = tmp_bcast_buf[1];
/* Exchange node id and Helper ranks among world processes */
ranks_in_world = calloc(nprocs, sizeof(int));
ranks_in_user_world = calloc(nprocs, sizeof(int));
for (i = 0; i < nprocs; i++) {
ranks_in_world[i] = i;
}
mpi_errno = PMPI_Group_translate_ranks(MTCORE_GROUP_WORLD, nprocs,
ranks_in_world, MTCORE_GROUP_USER_WORLD,
ranks_in_user_world);
if (mpi_errno != MPI_SUCCESS)
goto fn_fail;
MTCORE_ALL_NODE_IDS = calloc(nprocs, sizeof(int));
MTCORE_ALL_H_RANKS_IN_WORLD = calloc(user_nprocs * MTCORE_ENV.num_h, sizeof(int));
MTCORE_ALL_UNIQUE_H_RANKS_IN_WORLD = calloc(MTCORE_NUM_NODES * MTCORE_ENV.num_h, sizeof(int));
tmp_gather_buf = calloc(nprocs * (1 + MTCORE_ENV.num_h), sizeof(int));
tmp_gather_buf[rank * (1 + MTCORE_ENV.num_h)] = MTCORE_MY_NODE_ID;
for (i = 0; i < MTCORE_ENV.num_h; i++) {
tmp_gather_buf[rank * (1 + MTCORE_ENV.num_h) + i + 1] = MTCORE_H_RANKS_IN_WORLD[i];
}
mpi_errno = PMPI_Allgather(MPI_IN_PLACE, 0, MPI_DATATYPE_NULL,
tmp_gather_buf, 1 + MTCORE_ENV.num_h, MPI_INT, MPI_COMM_WORLD);
if (mpi_errno != MPI_SUCCESS)
goto fn_fail;
for (i = 0; i < nprocs; i++) {
int i_user_rank = 0;
node_id = tmp_gather_buf[i * (1 + MTCORE_ENV.num_h)];
MTCORE_ALL_NODE_IDS[i] = node_id;
/* Only copy helper ranks for user processes */
i_user_rank = ranks_in_user_world[i];
if (i_user_rank != MPI_UNDEFINED) {
for (j = 0; j < MTCORE_ENV.num_h; j++) {
MTCORE_ALL_H_RANKS_IN_WORLD[i_user_rank * MTCORE_ENV.num_h + j] =
tmp_gather_buf[i * (1 + MTCORE_ENV.num_h) + j + 1];
MTCORE_ALL_UNIQUE_H_RANKS_IN_WORLD[node_id * MTCORE_ENV.num_h + j] =
tmp_gather_buf[i * (1 + MTCORE_ENV.num_h) + j + 1];
}
}
}
#ifdef DEBUG
MTCORE_DBG_PRINT("Debug gathered info ***** \n");
for (i = 0; i < nprocs; i++) {
MTCORE_DBG_PRINT("node_id[%d]: %d\n", i, MTCORE_ALL_NODE_IDS[i]);
}
#endif
/* USER processes */
if (local_rank >= MTCORE_ENV.num_h) {
/* Get user ranks in world */
for (i = 0; i < user_nprocs; i++)
ranks_in_user_world[i] = i;
MTCORE_USER_RANKS_IN_WORLD = calloc(user_nprocs, sizeof(int));
mpi_errno = PMPI_Group_translate_ranks(MTCORE_GROUP_USER_WORLD, user_nprocs,
ranks_in_user_world, MTCORE_GROUP_WORLD,
MTCORE_USER_RANKS_IN_WORLD);
if (mpi_errno != MPI_SUCCESS)
goto fn_fail;
#ifdef DEBUG
for (i = 0; i < user_nprocs; i++) {
MTCORE_DBG_PRINT("helper_rank_in_world[%d]:\n", i);
for (j = 0; j < MTCORE_ENV.num_h; j++) {
MTCORE_DBG_PRINT(" %d\n", MTCORE_ALL_H_RANKS_IN_WORLD[i * MTCORE_ENV.num_h + j]);
}
}
#endif
MTCORE_DBG_PRINT("I am user, %d/%d in world, %d/%d in local, %d/%d in user world, "
"%d/%d in user local, node_id %d\n", rank, nprocs, local_rank,
local_nprocs, user_rank, user_nprocs, local_user_rank,
local_user_nprocs, MTCORE_MY_NODE_ID);
MTCORE_Init_win_cache();
}
/* Helper processes */
/* TODO: Helper process should not run user program */
else {
/* free local buffers before enter helper main function */
if (tmp_gather_buf)
free(tmp_gather_buf);
if (ranks_in_user_world)
free(ranks_in_user_world);
if (ranks_in_world)
free(ranks_in_world);
MTCORE_DBG_PRINT("I am helper, %d/%d in world, %d/%d in local, node_id %d\n", rank,
nprocs, local_rank, local_nprocs, MTCORE_MY_NODE_ID);
run_h_main();
exit(0);
}
fn_exit:
if (tmp_gather_buf)
free(tmp_gather_buf);
if (ranks_in_user_world)
free(ranks_in_user_world);
if (ranks_in_world)
free(ranks_in_world);
return mpi_errno;
fn_fail:
/* --BEGIN ERROR HANDLING-- */
if (MTCORE_COMM_USER_WORLD != MPI_COMM_NULL) {
MTCORE_DBG_PRINT("free MTCORE_COMM_USER_WORLD\n");
PMPI_Comm_free(&MTCORE_COMM_USER_WORLD);
}
if (MTCORE_COMM_LOCAL != MPI_COMM_NULL) {
MTCORE_DBG_PRINT("free MTCORE_COMM_LOCAL\n");
PMPI_Comm_free(&MTCORE_COMM_LOCAL);
}
if (MTCORE_COMM_USER_LOCAL != MPI_COMM_NULL) {
MTCORE_DBG_PRINT("free MTCORE_COMM_USER_LOCAL\n");
PMPI_Comm_free(&MTCORE_COMM_USER_LOCAL);
}
if (MTCORE_COMM_UR_WORLD != MPI_COMM_NULL) {
MTCORE_DBG_PRINT("free MTCORE_COMM_UR_WORLD\n");
PMPI_Comm_free(&MTCORE_COMM_UR_WORLD);
}
if (MTCORE_COMM_HELPER_LOCAL != MPI_COMM_NULL) {
MTCORE_DBG_PRINT("free MTCORE_COMM_HELPER_LOCAL\n");
PMPI_Comm_free(&MTCORE_COMM_HELPER_LOCAL);
}
if (MTCORE_GROUP_WORLD != MPI_GROUP_NULL)
PMPI_Group_free(&MTCORE_GROUP_WORLD);
if (MTCORE_GROUP_LOCAL != MPI_GROUP_NULL)
PMPI_Group_free(&MTCORE_GROUP_LOCAL);
if (MTCORE_GROUP_USER_WORLD != MPI_GROUP_NULL)
PMPI_Group_free(&MTCORE_GROUP_USER_WORLD);
if (MTCORE_H_RANKS_IN_WORLD)
free(MTCORE_H_RANKS_IN_WORLD);
if (MTCORE_H_RANKS_IN_LOCAL)
free(MTCORE_H_RANKS_IN_LOCAL);
if (MTCORE_ALL_H_RANKS_IN_WORLD)
free(MTCORE_ALL_H_RANKS_IN_WORLD);
if (MTCORE_ALL_UNIQUE_H_RANKS_IN_WORLD)
free(MTCORE_ALL_UNIQUE_H_RANKS_IN_WORLD);
if (MTCORE_ALL_NODE_IDS)
free(MTCORE_ALL_NODE_IDS);
if (MTCORE_USER_RANKS_IN_WORLD)
free(MTCORE_USER_RANKS_IN_WORLD);
MTCORE_Destroy_win_cache();
/* Reset global variables */
MTCORE_COMM_USER_WORLD = MPI_COMM_NULL;
MTCORE_COMM_USER_LOCAL = MPI_COMM_NULL;
MTCORE_COMM_LOCAL = MPI_COMM_NULL;
MTCORE_ALL_H_RANKS_IN_WORLD = NULL;
MTCORE_ALL_NODE_IDS = NULL;
PMPI_Abort(MPI_COMM_WORLD, 0);
goto fn_exit;
/* --END ERROR HANDLING-- */
}
