static inline int check_and_switch_target_state(MPIR_Win * win_ptr, MPIDI_RMA_Target_t * target,
int *is_able_to_issue, int *made_progress)
{
int rank = win_ptr->comm_ptr->rank;
int mpi_errno = MPI_SUCCESS;
(*made_progress) = 0;
(*is_able_to_issue) = 0;
if (target == NULL)
goto fn_exit;
/* When user event happens, move op in user pending list to network pending list */
if (target->sync.sync_flag == MPIDI_RMA_SYNC_FLUSH ||
target->sync.sync_flag == MPIDI_RMA_SYNC_FLUSH_LOCAL ||
target->sync.sync_flag == MPIDI_RMA_SYNC_UNLOCK || target->win_complete_flag) {
MPIDI_RMA_Op_t *user_op = target->pending_user_ops_list_head;
if (user_op != NULL) {
if (target->pending_net_ops_list_head == NULL)
win_ptr->num_targets_with_pending_net_ops++;
DL_DELETE(target->pending_user_ops_list_head, user_op);
DL_APPEND(target->pending_net_ops_list_head, user_op);
if (target->next_op_to_issue == NULL)
target->next_op_to_issue = user_op;
}
}
switch (target->access_state) {
case MPIDI_RMA_LOCK_CALLED:
if (target->sync.sync_flag == MPIDI_RMA_SYNC_NONE ||
target->sync.sync_flag == MPIDI_RMA_SYNC_FLUSH_LOCAL ||
target->sync.sync_flag == MPIDI_RMA_SYNC_FLUSH) {
if ((target->pending_net_ops_list_head == NULL ||
!target->pending_net_ops_list_head->piggyback_lock_candidate) &&
(target->pending_user_ops_list_head == NULL ||
!target->pending_user_ops_list_head->piggyback_lock_candidate)) {
/* issue lock request */
target->access_state = MPIDI_RMA_LOCK_ISSUED;
if (target->target_rank == rank) {
mpi_errno = acquire_local_lock(win_ptr, target->lock_type);
if (mpi_errno != MPI_SUCCESS)
MPIR_ERR_POP(mpi_errno);
}
else {
mpi_errno = send_lock_msg(target->target_rank, target->lock_type, win_ptr);
if (mpi_errno != MPI_SUCCESS)
MPIR_ERR_POP(mpi_errno);
}
(*made_progress) = 1;
}
}
else if (target->sync.sync_flag == MPIDI_RMA_SYNC_UNLOCK) {
if (target->pending_net_ops_list_head == NULL) {
/* No RMA operation has ever been posted to this target,
* finish issuing, no need to acquire the lock. Cleanup
* function will clean it up. */
target->access_state = MPIDI_RMA_LOCK_GRANTED;
/* We are done with ending synchronization, unset target's sync_flag. */
target->sync.sync_flag = MPIDI_RMA_SYNC_NONE;
(*made_progress) = 1;
}
else {
/* if we reach WIN_UNLOCK and there is still operation existing
* in pending list, this operation must be the only operation
* and it is prepared to piggyback LOCK and UNLOCK. */
MPIR_Assert(MPIR_CVAR_CH3_RMA_DELAY_ISSUING_FOR_PIGGYBACKING);
MPIR_Assert(target->pending_net_ops_list_head->next == NULL);
MPIR_Assert(target->pending_net_ops_list_head->piggyback_lock_candidate);
}
}
break;
case MPIDI_RMA_LOCK_GRANTED:
case MPIDI_RMA_NONE:
if (target->win_complete_flag) {
if (target->pending_net_ops_list_head == NULL) {
MPIDI_CH3_Pkt_flags_t flags = MPIDI_CH3_PKT_FLAG_NONE;
if (target->sync.sync_flag == MPIDI_RMA_SYNC_FLUSH &&
target->num_ops_flush_not_issued > 0) {
flags |= MPIDI_CH3_PKT_FLAG_RMA_FLUSH;
win_ptr->outstanding_acks++;
target->sync.outstanding_acks++;
target->num_ops_flush_not_issued = 0;
}
mpi_errno = send_decr_at_cnt_msg(target->target_rank, win_ptr, flags);
if (mpi_errno != MPI_SUCCESS)
MPIR_ERR_POP(mpi_errno);
/* We are done with ending synchronization, unset target's sync_flag. */
target->sync.sync_flag = MPIDI_RMA_SYNC_NONE;
(*made_progress) = 1;
}
}
else if (target->sync.sync_flag == MPIDI_RMA_SYNC_FLUSH) {
if (target->pending_net_ops_list_head == NULL) {
if (target->target_rank != rank) {
if (target->num_ops_flush_not_issued > 0) {
win_ptr->outstanding_acks++;
target->sync.outstanding_acks++;
target->num_ops_flush_not_issued = 0;
mpi_errno = send_flush_msg(target->target_rank, win_ptr);
if (mpi_errno != MPI_SUCCESS)
MPIR_ERR_POP(mpi_errno);
}
}
/* We are done with ending synchronization, unset target's sync_flag. */
target->sync.sync_flag = MPIDI_RMA_SYNC_NONE;
(*made_progress) = 1;
}
}
else if (target->sync.sync_flag == MPIDI_RMA_SYNC_UNLOCK) {
if (target->pending_net_ops_list_head == NULL) {
if (target->target_rank == rank) {
mpi_errno = MPIDI_CH3I_Release_lock(win_ptr);
if (mpi_errno != MPI_SUCCESS)
MPIR_ERR_POP(mpi_errno);
}
else {
MPIDI_CH3_Pkt_flags_t flag = MPIDI_CH3_PKT_FLAG_NONE;
if (target->num_ops_flush_not_issued == 0) {
flag = MPIDI_CH3_PKT_FLAG_RMA_UNLOCK_NO_ACK;
}
else {
win_ptr->outstanding_acks++;
target->sync.outstanding_acks++;
target->num_ops_flush_not_issued = 0;
}
mpi_errno = send_unlock_msg(target->target_rank, win_ptr, flag);
if (mpi_errno != MPI_SUCCESS)
MPIR_ERR_POP(mpi_errno);
}
/* We are done with ending synchronization, unset target's sync_flag. */
target->sync.sync_flag = MPIDI_RMA_SYNC_NONE;
(*made_progress) = 1;
}
}
break;
default:
break;
} /* end of switch */
if (target->access_state != MPIDI_RMA_LOCK_ISSUED) {
(*is_able_to_issue) = 1;
}
fn_exit:
return mpi_errno;
fn_fail:
goto fn_exit;
}
static inline int check_target_state(MPID_Win * win_ptr, MPIDI_RMA_Target_t * target,
int *made_progress)
{
int rank = win_ptr->comm_ptr->rank;
int mpi_errno = MPI_SUCCESS;
(*made_progress) = 0;
if (target == NULL)
goto fn_exit;
/* This check should only be performed when window-wide sync is finished, or
* current sync is per-target sync. */
if (win_ptr->states.access_state == MPIDI_RMA_NONE ||
win_ptr->states.access_state == MPIDI_RMA_FENCE_ISSUED ||
win_ptr->states.access_state == MPIDI_RMA_PSCW_ISSUED ||
win_ptr->states.access_state == MPIDI_RMA_LOCK_ALL_ISSUED) {
goto fn_exit;
}
switch (target->access_state) {
case MPIDI_RMA_LOCK_CALLED:
if (target->sync.sync_flag == MPIDI_RMA_SYNC_NONE ||
target->sync.sync_flag == MPIDI_RMA_SYNC_FLUSH_LOCAL ||
target->sync.sync_flag == MPIDI_RMA_SYNC_FLUSH) {
if (target->pending_op_list == NULL ||
!target->pending_op_list->piggyback_lock_candidate) {
/* issue lock request */
target->access_state = MPIDI_RMA_LOCK_ISSUED;
if (target->target_rank == rank) {
mpi_errno = acquire_local_lock(win_ptr, target->lock_type);
if (mpi_errno != MPI_SUCCESS)
MPIU_ERR_POP(mpi_errno);
}
else {
mpi_errno = send_lock_msg(target->target_rank, target->lock_type, win_ptr);
if (mpi_errno != MPI_SUCCESS)
MPIU_ERR_POP(mpi_errno);
}
(*made_progress) = 1;
}
}
else if (target->sync.sync_flag == MPIDI_RMA_SYNC_UNLOCK) {
if (target->pending_op_list == NULL) {
/* No RMA operation has ever been posted to this target,
* finish issuing, no need to acquire the lock. Cleanup
* function will clean it up. */
target->access_state = MPIDI_RMA_LOCK_GRANTED;
target->sync.outstanding_acks--;
MPIU_Assert(target->sync.outstanding_acks >= 0);
/* We are done with ending synchronization, unset target's sync_flag. */
target->sync.sync_flag = MPIDI_RMA_SYNC_NONE;
(*made_progress) = 1;
}
else {
/* if we reach WIN_UNLOCK and there is still operation existing
* in pending list, this operation must be the only operation
* and it is prepared to piggyback LOCK and UNLOCK. */
MPIU_Assert(target->pending_op_list->next == NULL);
MPIU_Assert(target->pending_op_list->piggyback_lock_candidate);
}
}
break;
case MPIDI_RMA_LOCK_GRANTED:
case MPIDI_RMA_NONE:
if (target->sync.sync_flag == MPIDI_RMA_SYNC_FLUSH) {
if (target->pending_op_list == NULL) {
if (target->target_rank == rank) {
target->sync.outstanding_acks--;
MPIU_Assert(target->sync.outstanding_acks >= 0);
}
else {
if (target->put_acc_issued) {
mpi_errno = send_flush_msg(target->target_rank, win_ptr);
if (mpi_errno != MPI_SUCCESS)
MPIU_ERR_POP(mpi_errno);
}
else {
/* We did not issue PUT/ACC since the last
* synchronization call, therefore here we
* don't need ACK back */
target->sync.outstanding_acks--;
MPIU_Assert(target->sync.outstanding_acks >= 0);
}
}
/* We are done with ending synchronization, unset target's sync_flag. */
target->sync.sync_flag = MPIDI_RMA_SYNC_NONE;
(*made_progress) = 1;
}
}
else if (target->sync.sync_flag == MPIDI_RMA_SYNC_UNLOCK) {
if (target->pending_op_list == NULL) {
if (target->target_rank == rank) {
target->sync.outstanding_acks--;
MPIU_Assert(target->sync.outstanding_acks >= 0);
mpi_errno = MPIDI_CH3I_Release_lock(win_ptr);
if (mpi_errno != MPI_SUCCESS)
MPIU_ERR_POP(mpi_errno);
}
else {
MPIDI_CH3_Pkt_flags_t flag = MPIDI_CH3_PKT_FLAG_NONE;
if (!target->put_acc_issued) {
/* We did not issue PUT/ACC since the last
* synchronization call, therefore here we
* don't need ACK back */
target->sync.outstanding_acks--;
MPIU_Assert(target->sync.outstanding_acks >= 0);
flag = MPIDI_CH3_PKT_FLAG_RMA_UNLOCK_NO_ACK;
}
mpi_errno = send_unlock_msg(target->target_rank, win_ptr, flag);
if (mpi_errno != MPI_SUCCESS)
MPIU_ERR_POP(mpi_errno);
}
/* We are done with ending synchronization, unset target's sync_flag. */
target->sync.sync_flag = MPIDI_RMA_SYNC_NONE;
(*made_progress) = 1;
}
}
break;
default:
break;
} /* end of switch */
fn_exit:
return mpi_errno;
fn_fail:
goto fn_exit;
}
