static int MTCORE_Complete_flush(int start_grp_size, MTCORE_Win * uh_win)
{
int mpi_errno = MPI_SUCCESS;
int user_rank, user_nprocs;
int i, j, k;
MTCORE_DBG_PRINT_FCNAME();
PMPI_Comm_rank(uh_win->user_comm, &user_rank);
/* Flush helpers to finish the sequence of locally issued RMA operations */
#ifdef MTCORE_ENABLE_SYNC_ALL_OPT
/* Optimization for MPI implementations that have optimized lock_all.
* However, user should be noted that, if MPI implementation issues lock messages
* for every target even if it does not have any operation, this optimization
* could lose performance and even lose asynchronous! */
MTCORE_DBG_PRINT("[%d]flush_all(active_win 0x%x)\n", user_rank, uh_win->active_win);
mpi_errno = PMPI_Win_flush_all(uh_win->active_win);
if (mpi_errno != MPI_SUCCESS)
goto fn_fail;
#else
/* Flush every helper once in the single window.
* TODO: track op issuing, only flush the helpers which receive ops. */
for (i = 0; i < uh_win->num_h_ranks_in_uh; i++) {
mpi_errno = PMPI_Win_flush(uh_win->h_ranks_in_uh[i], uh_win->active_win);
if (mpi_errno != MPI_SUCCESS)
goto fn_fail;
}
#ifdef MTCORE_ENABLE_LOCAL_LOCK_OPT
/* Need flush local target */
for (i = 0; i < start_grp_size; i++) {
if (uh_win->start_ranks_in_win_group[i] == user_rank) {
mpi_errno = PMPI_Win_flush(uh_win->my_rank_in_uh_comm, uh_win->active_win);
if (mpi_errno != MPI_SUCCESS)
goto fn_fail;
}
}
#endif
#endif
/* TODO: All the operations which we have not wrapped up will be failed, because they
* are issued to user window. We need wrap up all operations.
*/
fn_exit:
return mpi_errno;
fn_fail:
goto fn_exit;
}
int MPI_Win_flush_all(MPI_Win win)
{
int ret;
client_t* myclient = window_get_client(win);
cqueue_t* mycqueue = client_get_cqueue(myclient);
if (myclient == NULL)
ret = PMPI_Win_flush_all(win);
else
ret = cqueue_win_flush_all(mycqueue, win);
return ret;
}
int MPI_Win_flush_all_C_Wrapper (MPI_Win win)
{
int ierror;
TRACE_MPIEVENT (LAST_READ_TIME, MPI_WIN_FLUSH_ALL_EV, EVT_BEGIN, EMPTY, EMPTY,
EMPTY, EMPTY, EMPTY);
ierror = PMPI_Win_flush_all (win);
TRACE_MPIEVENT (LAST_READ_TIME, MPI_WIN_FLUSH_ALL_EV, EVT_END, EMPTY, EMPTY,
EMPTY, EMPTY, EMPTY);
updateStats_OTHER(global_mpi_stats);
return ierror;
}
int MPI_Win_flush_all(MPI_Win win)
{
MTCORE_Win *uh_win;
int mpi_errno = MPI_SUCCESS;
int user_rank, user_nprocs;
int i, j, k;
MTCORE_DBG_PRINT_FCNAME();
MTCORE_Fetch_uh_win_from_cache(win, uh_win);
if (uh_win == NULL) {
/* normal window */
return PMPI_Win_flush_all(win);
}
/* mtcore window starts */
MTCORE_Assert((uh_win->info_args.epoch_type & MTCORE_EPOCH_LOCK) ||
(uh_win->info_args.epoch_type & MTCORE_EPOCH_LOCK_ALL));
PMPI_Comm_rank(uh_win->user_comm, &user_rank);
PMPI_Comm_size(uh_win->user_comm, &user_nprocs);
if (!(uh_win->info_args.epoch_type & MTCORE_EPOCH_LOCK)) {
/* In lock_all only epoch, single window is shared by multiple targets. */
#ifdef MTCORE_ENABLE_SYNC_ALL_OPT
/* Optimization for MPI implementations that have optimized lock_all.
* However, user should be noted that, if MPI implementation issues lock messages
* for every target even if it does not have any operation, this optimization
* could lose performance and even lose asynchronous! */
MTCORE_DBG_PRINT("[%d]flush_all(uh_win 0x%x)\n", user_rank, uh_win->uh_wins[0]);
mpi_errno = PMPI_Win_flush_all(uh_win->uh_wins[0]);
if (mpi_errno != MPI_SUCCESS)
goto fn_fail;
#else
/* Flush every helper once in the single window.
* TODO: track op issuing, only flush the helpers which receive ops. */
for (i = 0; i < uh_win->num_h_ranks_in_uh; i++) {
mpi_errno = PMPI_Win_flush(uh_win->h_ranks_in_uh[i], uh_win->uh_wins[0]);
if (mpi_errno != MPI_SUCCESS)
goto fn_fail;
}
#endif
#ifdef MTCORE_ENABLE_LOCAL_LOCK_OPT
mpi_errno = MTCORE_Win_flush_self_impl(uh_win);
if (mpi_errno != MPI_SUCCESS)
goto fn_fail;
#endif
}
else {
/* In lock_all/lock mixed epoch, separate windows are bound with each target. */
mpi_errno = MTCORE_Win_mixed_flush_all_impl(win, uh_win);
if (mpi_errno != MPI_SUCCESS)
goto fn_fail;
}
#if defined(MTCORE_ENABLE_RUNTIME_LOAD_OPT)
for (i = 0; i < user_nprocs; i++) {
for (j = 0; j < uh_win->targets[i].num_segs; j++) {
/* Lock of main helper is granted, we can start load balancing from the next flush/unlock.
* Note that only target which was issued operations to is guaranteed to be granted. */
if (uh_win->targets[i].segs[j].main_lock_stat == MTCORE_MAIN_LOCK_OP_ISSUED) {
uh_win->targets[i].segs[j].main_lock_stat = MTCORE_MAIN_LOCK_GRANTED;
MTCORE_DBG_PRINT("[%d] main lock (rank %d, seg %d) granted\n", user_rank, i, j);
}
MTCORE_Reset_win_target_load_opt(i, uh_win);
}
}
#endif
/* TODO: All the operations which we have not wrapped up will be failed, because they
* are issued to user window. We need wrap up all operations.
*/
fn_exit:
return mpi_errno;
fn_fail:
goto fn_exit;
}
static int MTCORE_Win_mixed_flush_all_impl(MPI_Win win, MTCORE_Win * uh_win)
{
int mpi_errno = MPI_SUCCESS;
int user_rank, user_nprocs;
int i, j, k;
PMPI_Comm_rank(uh_win->user_comm, &user_rank);
PMPI_Comm_size(uh_win->user_comm, &user_nprocs);
/* Flush all Helpers in corresponding uh-window of each target process.. */
#ifdef MTCORE_ENABLE_SYNC_ALL_OPT
/* Optimization for MPI implementations that have optimized lock_all.
* However, user should be noted that, if MPI implementation issues lock messages
* for every target even if it does not have any operation, this optimization
* could lose performance and even lose asynchronous! */
for (i = 0; i < uh_win->num_uh_wins; i++) {
MTCORE_DBG_PRINT("[%d]flush_all(uh_win 0x%x)\n", user_rank, uh_win->uh_wins[i]);
mpi_errno = PMPI_Win_flush_all(uh_win->uh_wins[i]);
if (mpi_errno != MPI_SUCCESS)
goto fn_fail;
}
#else
/* TODO: track op issuing, only flush the helpers which receive ops. */
for (i = 0; i < user_nprocs; i++) {
#if !defined(MTCORE_ENABLE_RUNTIME_LOAD_OPT)
/* RMA operations are only issued to the main helper, so we only flush it. */
for (j = 0; j < uh_win->targets[i].num_segs; j++) {
int main_h_off = uh_win->targets[i].segs[j].main_h_off;
int target_h_rank_in_uh = uh_win->targets[i].h_ranks_in_uh[main_h_off];
MTCORE_DBG_PRINT("[%d]flush(Helper(%d), uh_wins 0x%x), instead of "
"target rank %d seg %d\n", user_rank, target_h_rank_in_uh,
uh_win->targets[i].segs[j].uh_win, i, j);
mpi_errno = PMPI_Win_flush(target_h_rank_in_uh, uh_win->targets[i].segs[j].uh_win);
if (mpi_errno != MPI_SUCCESS)
goto fn_fail;
}
#else
/* RMA operations may be distributed to all helpers, so we should
* flush all helpers on all windows. See discussion in win_flush. */
for (k = 0; k < MTCORE_ENV.num_h; k++) {
int target_h_rank_in_uh = uh_win->targets[i].h_ranks_in_uh[k];
MTCORE_DBG_PRINT("[%d]flush(Helper(%d), uh_win 0x%x), instead of "
"target rank %d\n", user_rank, target_h_rank_in_uh,
uh_win->targets[i].uh_win, i);
mpi_errno = PMPI_Win_flush(target_h_rank_in_uh, uh_win->targets[i].uh_win);
if (mpi_errno != MPI_SUCCESS)
goto fn_fail;
}
#endif /*end of MTCORE_ENABLE_RUNTIME_LOAD_OPT */
}
#endif /*end of MTCORE_ENABLE_SYNC_ALL_OPT */
#ifdef MTCORE_ENABLE_LOCAL_LOCK_OPT
mpi_errno = MTCORE_Win_flush_self_impl(uh_win);
if (mpi_errno != MPI_SUCCESS)
goto fn_fail;
#endif
fn_exit:
return mpi_errno;
fn_fail:
goto fn_exit;
}
int MPI_Win_flush(int target_rank, MPI_Win win)
{
MTCORE_Win *uh_win;
int mpi_errno = MPI_SUCCESS;
int user_rank;
int j, k;
MTCORE_DBG_PRINT_FCNAME();
MTCORE_Fetch_uh_win_from_cache(win, uh_win);
if (uh_win == NULL) {
/* normal window */
return PMPI_Win_flush(target_rank, win);
}
/* mtcore window starts */
MTCORE_Assert((uh_win->info_args.epoch_type & MTCORE_EPOCH_LOCK) ||
(uh_win->info_args.epoch_type & MTCORE_EPOCH_LOCK_ALL));
PMPI_Comm_rank(uh_win->user_comm, &user_rank);
#ifdef MTCORE_ENABLE_LOCAL_LOCK_OPT
if (user_rank == target_rank && uh_win->is_self_locked) {
/* If target is itself, also flush the target on local window.
* Local window is referred from another internal window in win_allocate.
* Note that global windows still need to be flushed because atomicity required
* operations (i.e., ACC and FOP) are still sent through global window.
*/
MTCORE_DBG_PRINT("[%d]flush self(%d, local win 0x%x)\n", user_rank,
uh_win->my_rank_in_uh_comm, uh_win->my_uh_win);
mpi_errno = PMPI_Win_flush(uh_win->my_rank_in_uh_comm, uh_win->my_uh_win);
if (mpi_errno != MPI_SUCCESS)
goto fn_fail;
}
#endif
{
#ifdef MTCORE_ENABLE_SYNC_ALL_OPT
/* Optimization for MPI implementations that have optimized lock_all.
* However, user should be noted that, if MPI implementation issues lock messages
* for every target even if it does not have any operation, this optimization
* could lose performance and even lose asynchronous! */
MTCORE_DBG_PRINT("[%d]flush_all(uh_win 0x%x), instead of target rank %d\n",
user_rank, uh_win->targets[target_rank].uh_win, target_rank);
mpi_errno = PMPI_Win_flush_all(uh_win->targets[target_rank].uh_win);
if (mpi_errno != MPI_SUCCESS)
goto fn_fail;
#else
#if !defined(MTCORE_ENABLE_RUNTIME_LOAD_OPT)
/* RMA operations are only issued to the main helper, so we only flush it. */
/* TODO: track op issuing, only flush the helpers which receive ops. */
for (j = 0; j < uh_win->targets[target_rank].num_segs; j++) {
int main_h_off = uh_win->targets[target_rank].segs[j].main_h_off;
int target_h_rank_in_uh = uh_win->targets[target_rank].h_ranks_in_uh[main_h_off];
MTCORE_DBG_PRINT("[%d]flush(Helper(%d), uh_wins 0x%x), instead of "
"target rank %d seg %d\n", user_rank, target_h_rank_in_uh,
uh_win->targets[target_rank].segs[j].uh_win, target_rank, j);
mpi_errno = PMPI_Win_flush(target_h_rank_in_uh,
uh_win->targets[target_rank].segs[j].uh_win);
if (mpi_errno != MPI_SUCCESS)
goto fn_fail;
}
#else
/* RMA operations may be distributed to all helpers, so we should
* flush all helpers on all windows.
*
* Note that some flushes could be eliminated before the main lock of a
* segment granted (see above). However, we have to loop all the segments
* in order to check each lock status, and we may flush the same helper
* on the same window twice if the lock is granted on that segment.
* i.e., flush (H0, win0) and (H1, win0) twice for seg0 and seg1.
*
* Consider flush does nothing if no operations on that target in most
* MPI implementation, simpler code is better */
j = 0;
for (k = 0; k < MTCORE_ENV.num_h; k++) {
int target_h_rank_in_uh = uh_win->targets[target_rank].h_ranks_in_uh[k];
MTCORE_DBG_PRINT("[%d]flush(Helper(%d), uh_wins 0x%x), instead of "
"target rank %d\n", user_rank, target_h_rank_in_uh,
uh_win->targets[target_rank].uh_win, target_rank);
mpi_errno = PMPI_Win_flush(target_h_rank_in_uh, uh_win->targets[target_rank].uh_win);
if (mpi_errno != MPI_SUCCESS)
goto fn_fail;
}
#endif /*end of MTCORE_ENABLE_RUNTIME_LOAD_OPT */
#endif /*end of MTCORE_ENABLE_SYNC_ALL_OPT */
}
#if defined(MTCORE_ENABLE_RUNTIME_LOAD_OPT)
for (j = 0; j < uh_win->targets[target_rank].num_segs; j++) {
/* Lock of main helper is granted, we can start load balancing from the next flush/unlock.
* Note that only target which was issued operations to is guaranteed to be granted. */
if (uh_win->targets[target_rank].segs[j].main_lock_stat == MTCORE_MAIN_LOCK_OP_ISSUED) {
uh_win->targets[target_rank].segs[j].main_lock_stat = MTCORE_MAIN_LOCK_GRANTED;
MTCORE_DBG_PRINT("[%d] main lock (rank %d, seg %d) granted\n", user_rank, target_rank,
j);
}
MTCORE_Reset_win_target_load_opt(target_rank, uh_win);
}
#endif
/* TODO: All the operations which we have not wrapped up will be failed, because they
* are issued to user window. We need wrap up all operations.
*/
fn_exit:
return mpi_errno;
fn_fail:
goto fn_exit;
}
