static inline int check_and_switch_target_state(MPIR_Win * win_ptr, MPIDI_RMA_Target_t * target, int *is_able_to_issue, int *made_progress) { int rank = win_ptr->comm_ptr->rank; int mpi_errno = MPI_SUCCESS; (*made_progress) = 0; (*is_able_to_issue) = 0; if (target == NULL) goto fn_exit; /* When user event happens, move op in user pending list to network pending list */ if (target->sync.sync_flag == MPIDI_RMA_SYNC_FLUSH || target->sync.sync_flag == MPIDI_RMA_SYNC_FLUSH_LOCAL || target->sync.sync_flag == MPIDI_RMA_SYNC_UNLOCK || target->win_complete_flag) { MPIDI_RMA_Op_t *user_op = target->pending_user_ops_list_head; if (user_op != NULL) { if (target->pending_net_ops_list_head == NULL) win_ptr->num_targets_with_pending_net_ops++; DL_DELETE(target->pending_user_ops_list_head, user_op); DL_APPEND(target->pending_net_ops_list_head, user_op); if (target->next_op_to_issue == NULL) target->next_op_to_issue = user_op; } } switch (target->access_state) { case MPIDI_RMA_LOCK_CALLED: if (target->sync.sync_flag == MPIDI_RMA_SYNC_NONE || target->sync.sync_flag == MPIDI_RMA_SYNC_FLUSH_LOCAL || target->sync.sync_flag == MPIDI_RMA_SYNC_FLUSH) { if ((target->pending_net_ops_list_head == NULL || !target->pending_net_ops_list_head->piggyback_lock_candidate) && (target->pending_user_ops_list_head == NULL || !target->pending_user_ops_list_head->piggyback_lock_candidate)) { /* issue lock request */ target->access_state = MPIDI_RMA_LOCK_ISSUED; if (target->target_rank == rank) { mpi_errno = acquire_local_lock(win_ptr, target->lock_type); if (mpi_errno != MPI_SUCCESS) MPIR_ERR_POP(mpi_errno); } else { mpi_errno = send_lock_msg(target->target_rank, target->lock_type, win_ptr); if (mpi_errno != MPI_SUCCESS) MPIR_ERR_POP(mpi_errno); } (*made_progress) = 1; } } else if (target->sync.sync_flag == MPIDI_RMA_SYNC_UNLOCK) { if (target->pending_net_ops_list_head == NULL) { /* No RMA operation has ever been posted to this target, * finish issuing, no need to acquire the lock. Cleanup * function will clean it up. */ target->access_state = MPIDI_RMA_LOCK_GRANTED; /* We are done with ending synchronization, unset target's sync_flag. */ target->sync.sync_flag = MPIDI_RMA_SYNC_NONE; (*made_progress) = 1; } else { /* if we reach WIN_UNLOCK and there is still operation existing * in pending list, this operation must be the only operation * and it is prepared to piggyback LOCK and UNLOCK. */ MPIR_Assert(MPIR_CVAR_CH3_RMA_DELAY_ISSUING_FOR_PIGGYBACKING); MPIR_Assert(target->pending_net_ops_list_head->next == NULL); MPIR_Assert(target->pending_net_ops_list_head->piggyback_lock_candidate); } } break; case MPIDI_RMA_LOCK_GRANTED: case MPIDI_RMA_NONE: if (target->win_complete_flag) { if (target->pending_net_ops_list_head == NULL) { MPIDI_CH3_Pkt_flags_t flags = MPIDI_CH3_PKT_FLAG_NONE; if (target->sync.sync_flag == MPIDI_RMA_SYNC_FLUSH && target->num_ops_flush_not_issued > 0) { flags |= MPIDI_CH3_PKT_FLAG_RMA_FLUSH; win_ptr->outstanding_acks++; target->sync.outstanding_acks++; target->num_ops_flush_not_issued = 0; } mpi_errno = send_decr_at_cnt_msg(target->target_rank, win_ptr, flags); if (mpi_errno != MPI_SUCCESS) MPIR_ERR_POP(mpi_errno); /* We are done with ending synchronization, unset target's sync_flag. */ target->sync.sync_flag = MPIDI_RMA_SYNC_NONE; (*made_progress) = 1; } } else if (target->sync.sync_flag == MPIDI_RMA_SYNC_FLUSH) { if (target->pending_net_ops_list_head == NULL) { if (target->target_rank != rank) { if (target->num_ops_flush_not_issued > 0) { win_ptr->outstanding_acks++; target->sync.outstanding_acks++; target->num_ops_flush_not_issued = 0; mpi_errno = send_flush_msg(target->target_rank, win_ptr); if (mpi_errno != MPI_SUCCESS) MPIR_ERR_POP(mpi_errno); } } /* We are done with ending synchronization, unset target's sync_flag. */ target->sync.sync_flag = MPIDI_RMA_SYNC_NONE; (*made_progress) = 1; } } else if (target->sync.sync_flag == MPIDI_RMA_SYNC_UNLOCK) { if (target->pending_net_ops_list_head == NULL) { if (target->target_rank == rank) { mpi_errno = MPIDI_CH3I_Release_lock(win_ptr); if (mpi_errno != MPI_SUCCESS) MPIR_ERR_POP(mpi_errno); } else { MPIDI_CH3_Pkt_flags_t flag = MPIDI_CH3_PKT_FLAG_NONE; if (target->num_ops_flush_not_issued == 0) { flag = MPIDI_CH3_PKT_FLAG_RMA_UNLOCK_NO_ACK; } else { win_ptr->outstanding_acks++; target->sync.outstanding_acks++; target->num_ops_flush_not_issued = 0; } mpi_errno = send_unlock_msg(target->target_rank, win_ptr, flag); if (mpi_errno != MPI_SUCCESS) MPIR_ERR_POP(mpi_errno); } /* We are done with ending synchronization, unset target's sync_flag. */ target->sync.sync_flag = MPIDI_RMA_SYNC_NONE; (*made_progress) = 1; } } break; default: break; } /* end of switch */ if (target->access_state != MPIDI_RMA_LOCK_ISSUED) { (*is_able_to_issue) = 1; } fn_exit: return mpi_errno; fn_fail: goto fn_exit; }
static inline int check_target_state(MPID_Win * win_ptr, MPIDI_RMA_Target_t * target, int *made_progress) { int rank = win_ptr->comm_ptr->rank; int mpi_errno = MPI_SUCCESS; (*made_progress) = 0; if (target == NULL) goto fn_exit; /* This check should only be performed when window-wide sync is finished, or * current sync is per-target sync. */ if (win_ptr->states.access_state == MPIDI_RMA_NONE || win_ptr->states.access_state == MPIDI_RMA_FENCE_ISSUED || win_ptr->states.access_state == MPIDI_RMA_PSCW_ISSUED || win_ptr->states.access_state == MPIDI_RMA_LOCK_ALL_ISSUED) { goto fn_exit; } switch (target->access_state) { case MPIDI_RMA_LOCK_CALLED: if (target->sync.sync_flag == MPIDI_RMA_SYNC_NONE || target->sync.sync_flag == MPIDI_RMA_SYNC_FLUSH_LOCAL || target->sync.sync_flag == MPIDI_RMA_SYNC_FLUSH) { if (target->pending_op_list == NULL || !target->pending_op_list->piggyback_lock_candidate) { /* issue lock request */ target->access_state = MPIDI_RMA_LOCK_ISSUED; if (target->target_rank == rank) { mpi_errno = acquire_local_lock(win_ptr, target->lock_type); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); } else { mpi_errno = send_lock_msg(target->target_rank, target->lock_type, win_ptr); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); } (*made_progress) = 1; } } else if (target->sync.sync_flag == MPIDI_RMA_SYNC_UNLOCK) { if (target->pending_op_list == NULL) { /* No RMA operation has ever been posted to this target, * finish issuing, no need to acquire the lock. Cleanup * function will clean it up. */ target->access_state = MPIDI_RMA_LOCK_GRANTED; target->sync.outstanding_acks--; MPIU_Assert(target->sync.outstanding_acks >= 0); /* We are done with ending synchronization, unset target's sync_flag. */ target->sync.sync_flag = MPIDI_RMA_SYNC_NONE; (*made_progress) = 1; } else { /* if we reach WIN_UNLOCK and there is still operation existing * in pending list, this operation must be the only operation * and it is prepared to piggyback LOCK and UNLOCK. */ MPIU_Assert(target->pending_op_list->next == NULL); MPIU_Assert(target->pending_op_list->piggyback_lock_candidate); } } break; case MPIDI_RMA_LOCK_GRANTED: case MPIDI_RMA_NONE: if (target->sync.sync_flag == MPIDI_RMA_SYNC_FLUSH) { if (target->pending_op_list == NULL) { if (target->target_rank == rank) { target->sync.outstanding_acks--; MPIU_Assert(target->sync.outstanding_acks >= 0); } else { if (target->put_acc_issued) { mpi_errno = send_flush_msg(target->target_rank, win_ptr); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); } else { /* We did not issue PUT/ACC since the last * synchronization call, therefore here we * don't need ACK back */ target->sync.outstanding_acks--; MPIU_Assert(target->sync.outstanding_acks >= 0); } } /* We are done with ending synchronization, unset target's sync_flag. */ target->sync.sync_flag = MPIDI_RMA_SYNC_NONE; (*made_progress) = 1; } } else if (target->sync.sync_flag == MPIDI_RMA_SYNC_UNLOCK) { if (target->pending_op_list == NULL) { if (target->target_rank == rank) { target->sync.outstanding_acks--; MPIU_Assert(target->sync.outstanding_acks >= 0); mpi_errno = MPIDI_CH3I_Release_lock(win_ptr); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); } else { MPIDI_CH3_Pkt_flags_t flag = MPIDI_CH3_PKT_FLAG_NONE; if (!target->put_acc_issued) { /* We did not issue PUT/ACC since the last * synchronization call, therefore here we * don't need ACK back */ target->sync.outstanding_acks--; MPIU_Assert(target->sync.outstanding_acks >= 0); flag = MPIDI_CH3_PKT_FLAG_RMA_UNLOCK_NO_ACK; } mpi_errno = send_unlock_msg(target->target_rank, win_ptr, flag); if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno); } /* We are done with ending synchronization, unset target's sync_flag. */ target->sync.sync_flag = MPIDI_RMA_SYNC_NONE; (*made_progress) = 1; } } break; default: break; } /* end of switch */ fn_exit: return mpi_errno; fn_fail: goto fn_exit; }