int MPI_Comm_free(MPI_Comm* comm)
{
int ret = MPI_SUCCESS;
if (max_ep > 0)
{
int commtype = handle_get_type(*comm);
handle_release(*comm);
if (commtype == COMM_REG)
ret = PMPI_Comm_free(comm);
}
else
ret = PMPI_Comm_free(comm);
return ret;
}
/**
* Measurement wrapper for MPI_Comm_free
* @note Manually adapted wrapper
* @note C interface
* @note Introduced with MPI 1.0
* @ingroup cg
*/
int MPI_Comm_free(MPI_Comm* comm)
{
const int event_gen_active = IS_EVENT_GEN_ON_FOR(CG);
int return_val;
if (event_gen_active)
{
EVENT_GEN_OFF();
esd_enter(epk_mpi_regid[EPK__MPI_COMM_FREE]);
}
epk_comm_free(*comm);
return_val = PMPI_Comm_free(comm);
if (event_gen_active)
{
esd_exit(epk_mpi_regid[EPK__MPI_COMM_FREE]);
EVENT_GEN_ON();
}
return return_val;
}
int oshmem_shmem_finalize(void)
{
int ret = OSHMEM_SUCCESS;
static int32_t finalize_has_already_started = 0;
if (opal_atomic_bool_cmpset_32(&finalize_has_already_started, 0, 1)
&& oshmem_shmem_initialized && !oshmem_shmem_aborted) {
/* Should be called first because ompi_mpi_finalize makes orte and opal finalization */
ret = _shmem_finalize();
if ((OSHMEM_SUCCESS == ret) && ompi_mpi_initialized
&& !ompi_mpi_finalized) {
PMPI_Comm_free(&oshmem_comm_world);
ret = ompi_mpi_finalize();
}
if (OSHMEM_SUCCESS == ret) {
oshmem_shmem_initialized = false;
}
}
return ret;
}
void vt_sync(MPI_Comm comm, uint64_t* ltime, int64_t* offset)
{
VT_MPI_INT myrank, myrank_host, myrank_sync;
VT_MPI_INT numnodes;
uint64_t time;
MPI_Comm host_comm;
MPI_Comm sync_comm;
VT_SUSPEND_IO_TRACING(VT_CURRENT_THREAD);
/* mark begin of clock synchronization */
time = vt_pform_wtime();
vt_enter(VT_CURRENT_THREAD, &time, vt_trc_regid[VT__TRC_SYNCTIME]);
/* barrier at entry */
PMPI_Barrier(comm);
*offset = 0;
*ltime = vt_pform_wtime();
PMPI_Comm_rank(comm, &myrank);
/* create communicator containing all processes on the same node */
PMPI_Comm_split(comm, (vt_pform_node_id() & 0x7FFFFFFF), 0, &host_comm);
PMPI_Comm_rank(host_comm, &myrank_host);
/* create communicator containing all processes with rank zero in the
previously created communicators */
PMPI_Comm_split(comm, myrank_host, 0, &sync_comm);
PMPI_Comm_rank(sync_comm, &myrank_sync);
PMPI_Comm_size(sync_comm, &numnodes);
/* measure offsets between all nodes and the root node (rank 0 in sync_comm) */
if (myrank_host == 0)
{
VT_MPI_INT i;
for (i = 1; i < numnodes; i++)
{
PMPI_Barrier(sync_comm);
if (myrank_sync == i)
*offset = sync_slave(ltime, 0, sync_comm);
else if (myrank_sync == 0)
*offset = sync_master(ltime, i, sync_comm);
}
}
/* distribute offset and ltime across all processes on the same node */
PMPI_Bcast(offset, 1, MPI_LONG_LONG_INT, 0, host_comm);
PMPI_Bcast(ltime, 1, MPI_LONG_LONG_INT, 0, host_comm);
PMPI_Comm_free(&host_comm);
PMPI_Comm_free(&sync_comm);
/* barrier at exit */
PMPI_Barrier(comm);
/* mark end of clock synchronization */
time = vt_pform_wtime();
vt_exit(VT_CURRENT_THREAD, &time);
VT_RESUME_IO_TRACING(VT_CURRENT_THREAD);
}
int MPI_Comm_free(MPI_Comm * comm)
{
return PMPI_Comm_free(comm);
}
int MPI_Win_free(MPI_Win * win)
{
static const char FCNAME[] = "MTCORE_Win_free";
int mpi_errno = MPI_SUCCESS;
MTCORE_Win *uh_win;
int user_rank, user_nprocs, user_local_rank, user_local_nprocs;
int i, j;
MPI_Request *reqs = NULL;
MPI_Status *stats = NULL;
MTCORE_DBG_PRINT_FCNAME();
MTCORE_Fetch_uh_win_from_cache(*win, uh_win);
if (uh_win == NULL) {
/* normal window */
return PMPI_Win_free(win);
}
/* mtcore window starts */
PMPI_Comm_rank(uh_win->user_comm, &user_rank);
PMPI_Comm_size(uh_win->user_comm, &user_nprocs);
PMPI_Comm_rank(uh_win->local_user_comm, &user_local_rank);
PMPI_Comm_size(uh_win->local_user_comm, &user_local_nprocs);
/* First unlock global active window */
if ((uh_win->info_args.epoch_type & MTCORE_EPOCH_FENCE) ||
(uh_win->info_args.epoch_type & MTCORE_EPOCH_PSCW)) {
MTCORE_DBG_PRINT("[%d]unlock_all(active_win 0x%x)\n", user_rank, uh_win->active_win);
/* Since all processes must be in win_free, we do not need worry
* the possibility losing asynchronous progress. */
mpi_errno = PMPI_Win_unlock_all(uh_win->active_win);
if (mpi_errno != MPI_SUCCESS)
goto fn_fail;
}
if (user_local_rank == 0) {
MTCORE_Func_start(MTCORE_FUNC_WIN_FREE, user_nprocs, user_local_nprocs);
}
/* Notify the handle of target Helper win. It is noted that helpers cannot
* fetch the corresponding window without handlers so that only global communicator
* can be used here.*/
if (user_local_rank == 0) {
reqs = calloc(MTCORE_ENV.num_h, sizeof(MPI_Request));
stats = calloc(MTCORE_ENV.num_h, sizeof(MPI_Status));
for (j = 0; j < MTCORE_ENV.num_h; j++) {
mpi_errno = PMPI_Isend(&uh_win->h_win_handles[j], 1, MPI_UNSIGNED_LONG,
MTCORE_H_RANKS_IN_LOCAL[j], 0, MTCORE_COMM_LOCAL, &reqs[j]);
}
mpi_errno = PMPI_Waitall(MTCORE_ENV.num_h, reqs, stats);
if (mpi_errno != MPI_SUCCESS)
goto fn_fail;
}
/* Free uh_win before local_uh_win, because all the incoming operations
* should be done before free shared buffers.
*
* We do not need additional barrier in Manticore for waiting all
* operations complete, because Win_free already internally add a barrier
* for waiting operations on that window complete.
*/
if (uh_win->num_uh_wins > 0 && uh_win->uh_wins) {
MTCORE_DBG_PRINT("\t free uh windows\n");
for (i = 0; i < uh_win->num_uh_wins; i++) {
if (uh_win->uh_wins[i]) {
mpi_errno = PMPI_Win_free(&uh_win->uh_wins[i]);
if (mpi_errno != MPI_SUCCESS)
goto fn_fail;
}
}
}
if (uh_win->active_win) {
MTCORE_DBG_PRINT("\t free active window\n");
mpi_errno = PMPI_Win_free(&uh_win->active_win);
if (mpi_errno != MPI_SUCCESS)
goto fn_fail;
}
if (uh_win->local_uh_win) {
MTCORE_DBG_PRINT("\t free shared window\n");
mpi_errno = PMPI_Win_free(&uh_win->local_uh_win);
if (mpi_errno != MPI_SUCCESS)
goto fn_fail;
}
if (uh_win->user_group != MPI_GROUP_NULL) {
mpi_errno = PMPI_Group_free(&uh_win->user_group);
if (mpi_errno != MPI_SUCCESS)
goto fn_fail;
}
if (uh_win->ur_h_comm && uh_win->ur_h_comm != MPI_COMM_NULL) {
MTCORE_DBG_PRINT("\t free user root + helpers communicator\n");
mpi_errno = PMPI_Comm_free(&uh_win->ur_h_comm);
if (mpi_errno != MPI_SUCCESS)
goto fn_fail;
}
if (uh_win->local_uh_comm && uh_win->local_uh_comm != MTCORE_COMM_LOCAL) {
MTCORE_DBG_PRINT("\t free shared communicator\n");
mpi_errno = PMPI_Comm_free(&uh_win->local_uh_comm);
if (mpi_errno != MPI_SUCCESS)
goto fn_fail;
}
if (uh_win->local_uh_group != MPI_GROUP_NULL) {
mpi_errno = PMPI_Group_free(&uh_win->local_uh_group);
if (mpi_errno != MPI_SUCCESS)
goto fn_fail;
}
if (uh_win->uh_comm != MPI_COMM_NULL && uh_win->uh_comm != MPI_COMM_WORLD) {
MTCORE_DBG_PRINT("\t free uh communicator\n");
mpi_errno = PMPI_Comm_free(&uh_win->uh_comm);
if (mpi_errno != MPI_SUCCESS)
goto fn_fail;
}
if (uh_win->uh_group != MPI_GROUP_NULL) {
mpi_errno = PMPI_Group_free(&uh_win->uh_group);
if (mpi_errno != MPI_SUCCESS)
goto fn_fail;
}
if (uh_win->local_user_comm && uh_win->local_user_comm != MTCORE_COMM_USER_LOCAL) {
MTCORE_DBG_PRINT("\t free local USER communicator\n");
mpi_errno = PMPI_Comm_free(&uh_win->local_user_comm);
if (mpi_errno != MPI_SUCCESS)
goto fn_fail;
}
if (uh_win->user_root_comm && uh_win->user_root_comm != MTCORE_COMM_UR_WORLD) {
MTCORE_DBG_PRINT("\t free ur communicator\n");
mpi_errno = PMPI_Comm_free(&uh_win->user_root_comm);
if (mpi_errno != MPI_SUCCESS)
goto fn_fail;
}
MTCORE_DBG_PRINT("\t free window cache\n");
MTCORE_Remove_uh_win_from_cache(*win);
MTCORE_DBG_PRINT("\t free user window\n");
mpi_errno = PMPI_Win_free(win);
if (mpi_errno != MPI_SUCCESS)
goto fn_fail;
/* free PSCW array in case use does not call complete/wait. */
if (uh_win->start_ranks_in_win_group)
free(uh_win->start_ranks_in_win_group);
if (uh_win->post_ranks_in_win_group)
free(uh_win->post_ranks_in_win_group);
/* uh_win->user_comm is created by user, will be freed by user. */
#if defined(MTCORE_ENABLE_RUNTIME_LOAD_OPT)
if (uh_win->h_ops_counts)
free(uh_win->h_ops_counts);
if (uh_win->h_bytes_counts)
free(uh_win->h_bytes_counts);
#endif
if (uh_win->targets) {
for (i = 0; i < user_nprocs; i++) {
if (uh_win->targets[i].base_h_offsets)
free(uh_win->targets[i].base_h_offsets);
if (uh_win->targets[i].h_ranks_in_uh)
free(uh_win->targets[i].h_ranks_in_uh);
if (uh_win->targets[i].segs)
free(uh_win->targets[i].segs);
}
free(uh_win->targets);
}
if (uh_win->h_ranks_in_uh)
free(uh_win->h_ranks_in_uh);
if (uh_win->h_win_handles)
free(uh_win->h_win_handles);
if (uh_win->uh_wins)
free(uh_win->uh_wins);
free(uh_win);
MTCORE_DBG_PRINT("Freed MTCORE window 0x%x\n", *win);
fn_exit:
if (reqs)
free(reqs);
if (stats)
free(stats);
return mpi_errno;
fn_fail:
goto fn_exit;
}
int MPI_Init_thread(int *argc, char ***argv, int required, int *provided)
{
int mpi_errno = MPI_SUCCESS;
int i, j;
int local_rank, local_nprocs, rank, nprocs, user_rank, user_nprocs;
int local_user_rank = -1, local_user_nprocs = -1;
int *tmp_gather_buf = NULL, node_id = 0;
int tmp_bcast_buf[2];
int *ranks_in_user_world = NULL, *ranks_in_world = NULL;
MTCORE_DBG_PRINT_FCNAME();
if (required == 0 && provided == NULL) {
/* default init */
mpi_errno = PMPI_Init(argc, argv);
if (mpi_errno != MPI_SUCCESS)
goto fn_fail;
}
else {
/* user init thread */
mpi_errno = PMPI_Init_thread(argc, argv, required, provided);
if (mpi_errno != MPI_SUCCESS)
goto fn_fail;
}
PMPI_Comm_size(MPI_COMM_WORLD, &nprocs);
PMPI_Comm_rank(MPI_COMM_WORLD, &rank);
MTCORE_MY_RANK_IN_WORLD = rank;
mpi_errno = MTCORE_Initialize_env();
if (mpi_errno != MPI_SUCCESS)
goto fn_fail;
/* Get a communicator only containing processes with shared memory */
mpi_errno = PMPI_Comm_split_type(MPI_COMM_WORLD, MPI_COMM_TYPE_SHARED, 0,
MPI_INFO_NULL, &MTCORE_COMM_LOCAL);
if (mpi_errno != MPI_SUCCESS)
goto fn_fail;
/* Check number of helpers and number of processes */
PMPI_Comm_rank(MTCORE_COMM_LOCAL, &local_rank);
PMPI_Comm_size(MTCORE_COMM_LOCAL, &local_nprocs);
if (local_nprocs < 2) {
fprintf(stderr, "No user process found, please run with more than 2 process per node\n");
mpi_errno = -1;
goto fn_fail;
}
if (MTCORE_ENV.num_h < 1 || MTCORE_ENV.num_h >= local_nprocs) {
fprintf(stderr, "Wrong value of number of helpers, %d. lt 1 or ge %d.\n",
MTCORE_ENV.num_h, local_nprocs);
mpi_errno = -1;
goto fn_fail;
}
/* Specify the first N local processes to be Helper processes */
MTCORE_H_RANKS_IN_LOCAL = calloc(MTCORE_ENV.num_h, sizeof(int));
MTCORE_H_RANKS_IN_WORLD = calloc(MTCORE_ENV.num_h, sizeof(int));
for (i = 0; i < MTCORE_ENV.num_h; i++) {
MTCORE_H_RANKS_IN_LOCAL[i] = i;
}
mpi_errno = PMPI_Comm_group(MPI_COMM_WORLD, &MTCORE_GROUP_WORLD);
mpi_errno = PMPI_Comm_group(MTCORE_COMM_LOCAL, &MTCORE_GROUP_LOCAL);
mpi_errno = PMPI_Group_translate_ranks(MTCORE_GROUP_LOCAL, MTCORE_ENV.num_h,
MTCORE_H_RANKS_IN_LOCAL, MTCORE_GROUP_WORLD,
MTCORE_H_RANKS_IN_WORLD);
if (mpi_errno != MPI_SUCCESS)
goto fn_fail;
/* Create a user comm_world including all the users,
* user will access it instead of comm_world */
mpi_errno = PMPI_Comm_split(MPI_COMM_WORLD,
local_rank < MTCORE_ENV.num_h, 0, &MTCORE_COMM_USER_WORLD);
if (mpi_errno != MPI_SUCCESS)
goto fn_fail;
PMPI_Comm_size(MTCORE_COMM_USER_WORLD, &user_nprocs);
PMPI_Comm_rank(MTCORE_COMM_USER_WORLD, &user_rank);
PMPI_Comm_group(MTCORE_COMM_USER_WORLD, &MTCORE_GROUP_USER_WORLD);
/* Create a user comm_local */
mpi_errno = PMPI_Comm_split(MTCORE_COMM_LOCAL,
local_rank < MTCORE_ENV.num_h, 0, &MTCORE_COMM_USER_LOCAL);
if (mpi_errno != MPI_SUCCESS)
goto fn_fail;
/* Create a helper comm_local */
mpi_errno = PMPI_Comm_split(MTCORE_COMM_LOCAL,
local_rank < MTCORE_ENV.num_h, 1, &MTCORE_COMM_HELPER_LOCAL);
if (mpi_errno != MPI_SUCCESS)
goto fn_fail;
/* Exchange node id among local processes */
/* -Only users create a user root communicator for exchanging local informations
* between different nodes*/
if (local_rank >= MTCORE_ENV.num_h) {
PMPI_Comm_rank(MTCORE_COMM_USER_LOCAL, &local_user_rank);
PMPI_Comm_size(MTCORE_COMM_USER_LOCAL, &local_user_nprocs);
mpi_errno = PMPI_Comm_split(MTCORE_COMM_USER_WORLD,
local_user_rank == 0, 1, &MTCORE_COMM_UR_WORLD);
if (mpi_errno != MPI_SUCCESS)
goto fn_fail;
/* -Only user roots determine a node id for each USER processes */
if (local_user_rank == 0) {
PMPI_Comm_size(MTCORE_COMM_UR_WORLD, &MTCORE_NUM_NODES);
PMPI_Comm_rank(MTCORE_COMM_UR_WORLD, &MTCORE_MY_NODE_ID);
tmp_bcast_buf[0] = MTCORE_MY_NODE_ID;
tmp_bcast_buf[1] = MTCORE_NUM_NODES;
}
}
/* -User root broadcasts to other local processes */
PMPI_Bcast(tmp_bcast_buf, 2, MPI_INT, MTCORE_ENV.num_h, MTCORE_COMM_LOCAL);
MTCORE_MY_NODE_ID = tmp_bcast_buf[0];
MTCORE_NUM_NODES = tmp_bcast_buf[1];
/* Exchange node id and Helper ranks among world processes */
ranks_in_world = calloc(nprocs, sizeof(int));
ranks_in_user_world = calloc(nprocs, sizeof(int));
for (i = 0; i < nprocs; i++) {
ranks_in_world[i] = i;
}
mpi_errno = PMPI_Group_translate_ranks(MTCORE_GROUP_WORLD, nprocs,
ranks_in_world, MTCORE_GROUP_USER_WORLD,
ranks_in_user_world);
if (mpi_errno != MPI_SUCCESS)
goto fn_fail;
MTCORE_ALL_NODE_IDS = calloc(nprocs, sizeof(int));
MTCORE_ALL_H_RANKS_IN_WORLD = calloc(user_nprocs * MTCORE_ENV.num_h, sizeof(int));
MTCORE_ALL_UNIQUE_H_RANKS_IN_WORLD = calloc(MTCORE_NUM_NODES * MTCORE_ENV.num_h, sizeof(int));
tmp_gather_buf = calloc(nprocs * (1 + MTCORE_ENV.num_h), sizeof(int));
tmp_gather_buf[rank * (1 + MTCORE_ENV.num_h)] = MTCORE_MY_NODE_ID;
for (i = 0; i < MTCORE_ENV.num_h; i++) {
tmp_gather_buf[rank * (1 + MTCORE_ENV.num_h) + i + 1] = MTCORE_H_RANKS_IN_WORLD[i];
}
mpi_errno = PMPI_Allgather(MPI_IN_PLACE, 0, MPI_DATATYPE_NULL,
tmp_gather_buf, 1 + MTCORE_ENV.num_h, MPI_INT, MPI_COMM_WORLD);
if (mpi_errno != MPI_SUCCESS)
goto fn_fail;
for (i = 0; i < nprocs; i++) {
int i_user_rank = 0;
node_id = tmp_gather_buf[i * (1 + MTCORE_ENV.num_h)];
MTCORE_ALL_NODE_IDS[i] = node_id;
/* Only copy helper ranks for user processes */
i_user_rank = ranks_in_user_world[i];
if (i_user_rank != MPI_UNDEFINED) {
for (j = 0; j < MTCORE_ENV.num_h; j++) {
MTCORE_ALL_H_RANKS_IN_WORLD[i_user_rank * MTCORE_ENV.num_h + j] =
tmp_gather_buf[i * (1 + MTCORE_ENV.num_h) + j + 1];
MTCORE_ALL_UNIQUE_H_RANKS_IN_WORLD[node_id * MTCORE_ENV.num_h + j] =
tmp_gather_buf[i * (1 + MTCORE_ENV.num_h) + j + 1];
}
}
}
#ifdef DEBUG
MTCORE_DBG_PRINT("Debug gathered info ***** \n");
for (i = 0; i < nprocs; i++) {
MTCORE_DBG_PRINT("node_id[%d]: %d\n", i, MTCORE_ALL_NODE_IDS[i]);
}
#endif
/* USER processes */
if (local_rank >= MTCORE_ENV.num_h) {
/* Get user ranks in world */
for (i = 0; i < user_nprocs; i++)
ranks_in_user_world[i] = i;
MTCORE_USER_RANKS_IN_WORLD = calloc(user_nprocs, sizeof(int));
mpi_errno = PMPI_Group_translate_ranks(MTCORE_GROUP_USER_WORLD, user_nprocs,
ranks_in_user_world, MTCORE_GROUP_WORLD,
MTCORE_USER_RANKS_IN_WORLD);
if (mpi_errno != MPI_SUCCESS)
goto fn_fail;
#ifdef DEBUG
for (i = 0; i < user_nprocs; i++) {
MTCORE_DBG_PRINT("helper_rank_in_world[%d]:\n", i);
for (j = 0; j < MTCORE_ENV.num_h; j++) {
MTCORE_DBG_PRINT(" %d\n", MTCORE_ALL_H_RANKS_IN_WORLD[i * MTCORE_ENV.num_h + j]);
}
}
#endif
MTCORE_DBG_PRINT("I am user, %d/%d in world, %d/%d in local, %d/%d in user world, "
"%d/%d in user local, node_id %d\n", rank, nprocs, local_rank,
local_nprocs, user_rank, user_nprocs, local_user_rank,
local_user_nprocs, MTCORE_MY_NODE_ID);
MTCORE_Init_win_cache();
}
/* Helper processes */
/* TODO: Helper process should not run user program */
else {
/* free local buffers before enter helper main function */
if (tmp_gather_buf)
free(tmp_gather_buf);
if (ranks_in_user_world)
free(ranks_in_user_world);
if (ranks_in_world)
free(ranks_in_world);
MTCORE_DBG_PRINT("I am helper, %d/%d in world, %d/%d in local, node_id %d\n", rank,
nprocs, local_rank, local_nprocs, MTCORE_MY_NODE_ID);
run_h_main();
exit(0);
}
fn_exit:
if (tmp_gather_buf)
free(tmp_gather_buf);
if (ranks_in_user_world)
free(ranks_in_user_world);
if (ranks_in_world)
free(ranks_in_world);
return mpi_errno;
fn_fail:
/* --BEGIN ERROR HANDLING-- */
if (MTCORE_COMM_USER_WORLD != MPI_COMM_NULL) {
MTCORE_DBG_PRINT("free MTCORE_COMM_USER_WORLD\n");
PMPI_Comm_free(&MTCORE_COMM_USER_WORLD);
}
if (MTCORE_COMM_LOCAL != MPI_COMM_NULL) {
MTCORE_DBG_PRINT("free MTCORE_COMM_LOCAL\n");
PMPI_Comm_free(&MTCORE_COMM_LOCAL);
}
if (MTCORE_COMM_USER_LOCAL != MPI_COMM_NULL) {
MTCORE_DBG_PRINT("free MTCORE_COMM_USER_LOCAL\n");
PMPI_Comm_free(&MTCORE_COMM_USER_LOCAL);
}
if (MTCORE_COMM_UR_WORLD != MPI_COMM_NULL) {
MTCORE_DBG_PRINT("free MTCORE_COMM_UR_WORLD\n");
PMPI_Comm_free(&MTCORE_COMM_UR_WORLD);
}
if (MTCORE_COMM_HELPER_LOCAL != MPI_COMM_NULL) {
MTCORE_DBG_PRINT("free MTCORE_COMM_HELPER_LOCAL\n");
PMPI_Comm_free(&MTCORE_COMM_HELPER_LOCAL);
}
if (MTCORE_GROUP_WORLD != MPI_GROUP_NULL)
PMPI_Group_free(&MTCORE_GROUP_WORLD);
if (MTCORE_GROUP_LOCAL != MPI_GROUP_NULL)
PMPI_Group_free(&MTCORE_GROUP_LOCAL);
if (MTCORE_GROUP_USER_WORLD != MPI_GROUP_NULL)
PMPI_Group_free(&MTCORE_GROUP_USER_WORLD);
if (MTCORE_H_RANKS_IN_WORLD)
free(MTCORE_H_RANKS_IN_WORLD);
if (MTCORE_H_RANKS_IN_LOCAL)
free(MTCORE_H_RANKS_IN_LOCAL);
if (MTCORE_ALL_H_RANKS_IN_WORLD)
free(MTCORE_ALL_H_RANKS_IN_WORLD);
if (MTCORE_ALL_UNIQUE_H_RANKS_IN_WORLD)
free(MTCORE_ALL_UNIQUE_H_RANKS_IN_WORLD);
if (MTCORE_ALL_NODE_IDS)
free(MTCORE_ALL_NODE_IDS);
if (MTCORE_USER_RANKS_IN_WORLD)
free(MTCORE_USER_RANKS_IN_WORLD);
MTCORE_Destroy_win_cache();
/* Reset global variables */
MTCORE_COMM_USER_WORLD = MPI_COMM_NULL;
MTCORE_COMM_USER_LOCAL = MPI_COMM_NULL;
MTCORE_COMM_LOCAL = MPI_COMM_NULL;
MTCORE_ALL_H_RANKS_IN_WORLD = NULL;
MTCORE_ALL_NODE_IDS = NULL;
PMPI_Abort(MPI_COMM_WORLD, 0);
goto fn_exit;
/* --END ERROR HANDLING-- */
}
