static inline int issue_ops_target(MPIR_Win * win_ptr, MPIDI_RMA_Target_t * target,
int *made_progress)
{
MPIDI_RMA_Op_t *curr_op = NULL;
MPIDI_CH3_Pkt_flags_t flags;
int first_op = 1, mpi_errno = MPI_SUCCESS;
static int fn_reentrance_check = FALSE;
/* this function is not reentrant. if it is invoked in a
* reentrant manner, simply exit without doing anything. */
if (fn_reentrance_check == TRUE)
goto fn_exit;
fn_reentrance_check = TRUE;
(*made_progress) = 0;
if (win_ptr->num_targets_with_pending_net_ops == 0 || target == NULL ||
target->pending_net_ops_list_head == NULL)
goto finish_issue;
/* Issue out operations in the list. */
curr_op = target->next_op_to_issue;
while (curr_op != NULL) {
int op_completed = FALSE;
if (target->access_state == MPIDI_RMA_LOCK_ISSUED) {
/* It is possible that the previous OP+LOCK changes
* lock state to LOCK_ISSUED. */
break;
}
target->num_ops_flush_not_issued++;
flags = MPIDI_CH3_PKT_FLAG_NONE;
if (first_op) {
/* piggyback on first OP. */
if (target->access_state == MPIDI_RMA_LOCK_CALLED) {
MPIR_Assert(curr_op->piggyback_lock_candidate);
if (target->lock_type == MPI_LOCK_SHARED)
flags |= MPIDI_CH3_PKT_FLAG_RMA_LOCK_SHARED;
else {
MPIR_Assert(target->lock_type == MPI_LOCK_EXCLUSIVE);
flags |= MPIDI_CH3_PKT_FLAG_RMA_LOCK_EXCLUSIVE;
}
target->access_state = MPIDI_RMA_LOCK_ISSUED;
}
first_op = 0;
}
/* piggyback FLUSH on current OP if one of the following
* conditions meet:
* (1) ordered flush is not guaranteed;
* (2) operation is a READ op (GET, GACC, FOP, CAS) */
if ((!MPIDI_CH3U_Win_pkt_orderings.am_flush_ordered) ||
MPIDI_CH3I_RMA_PKT_IS_READ_OP(curr_op->pkt)) {
flags |= MPIDI_CH3_PKT_FLAG_RMA_FLUSH;
}
if (curr_op->next == NULL) {
/* piggyback on last OP. */
if (target->sync.sync_flag == MPIDI_RMA_SYNC_FLUSH) {
flags |= MPIDI_CH3_PKT_FLAG_RMA_FLUSH;
}
else if (target->sync.sync_flag == MPIDI_RMA_SYNC_UNLOCK) {
flags |= MPIDI_CH3_PKT_FLAG_RMA_UNLOCK;
/* if piggyback UNLOCK then unset FLUSH (set for every
* operation on out-of-order network). */
flags &= ~MPIDI_CH3_PKT_FLAG_RMA_FLUSH;
}
if (target->win_complete_flag)
flags |= MPIDI_CH3_PKT_FLAG_RMA_DECR_AT_COUNTER;
}
/* only increase ack counter when FLUSH or UNLOCK flag is set,
* but without LOCK piggyback. */
if (((flags & MPIDI_CH3_PKT_FLAG_RMA_FLUSH)
|| (flags & MPIDI_CH3_PKT_FLAG_RMA_UNLOCK))) {
win_ptr->outstanding_acks++;
target->sync.outstanding_acks++;
target->num_ops_flush_not_issued = 0;
}
mpi_errno = issue_rma_op(curr_op, win_ptr, target, flags);
if (mpi_errno != MPI_SUCCESS)
MPIR_ERR_POP(mpi_errno);
(*made_progress) = 1;
if (flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_SHARED ||
flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_EXCLUSIVE) {
/* If this operation is piggybacked with LOCK,
* do not move it out of pending list, and do
* not complete the user request, because we
* may need to re-transmit it. */
break;
}
target->next_op_to_issue = curr_op->next;
if (target->next_op_to_issue == NULL) {
if (((target->sync.sync_flag == MPIDI_RMA_SYNC_FLUSH) &&
(flags & MPIDI_CH3_PKT_FLAG_RMA_FLUSH)) ||
((target->sync.sync_flag == MPIDI_RMA_SYNC_UNLOCK) &&
(flags & MPIDI_CH3_PKT_FLAG_RMA_UNLOCK))) {
/* We are done with ending sync, unset target's sync_flag. */
target->sync.sync_flag = MPIDI_RMA_SYNC_NONE;
}
}
mpi_errno = check_and_set_req_completion(win_ptr, target, curr_op, &op_completed);
if (mpi_errno != MPI_SUCCESS) {
MPIR_ERR_POP(mpi_errno);
}
if (op_completed == FALSE) {
if (MPIDI_CH3I_RMA_Active_req_cnt > MPIR_CVAR_CH3_RMA_POKE_PROGRESS_REQ_THRESHOLD) {
mpi_errno = poke_progress_engine();
if (mpi_errno != MPI_SUCCESS)
MPIR_ERR_POP(mpi_errno);
}
}
curr_op = target->next_op_to_issue;
} /* end of while loop */
finish_issue:
fn_reentrance_check = FALSE;
fn_exit:
return mpi_errno;
fn_fail:
goto fn_exit;
}
static inline int issue_ops_target(MPID_Win * win_ptr, MPIDI_RMA_Target_t * target,
int *made_progress)
{
MPIDI_RMA_Op_t *curr_op = NULL;
MPIDI_CH3_Pkt_flags_t flags;
int first_op = 1, mpi_errno = MPI_SUCCESS;
(*made_progress) = 0;
if (win_ptr->non_empty_slots == 0 || target == NULL)
goto fn_exit;
/* Exit if window-wide sync is not finished */
if (win_ptr->states.access_state == MPIDI_RMA_NONE ||
win_ptr->states.access_state == MPIDI_RMA_FENCE_ISSUED ||
win_ptr->states.access_state == MPIDI_RMA_PSCW_ISSUED ||
win_ptr->states.access_state == MPIDI_RMA_LOCK_ALL_ISSUED)
goto fn_exit;
/* Exit if per-target sync is not finished */
if (win_ptr->states.access_state == MPIDI_RMA_PER_TARGET ||
win_ptr->states.access_state == MPIDI_RMA_LOCK_ALL_CALLED) {
if (target->access_state == MPIDI_RMA_LOCK_ISSUED)
goto fn_exit;
}
/* Issue out operations in the list. */
curr_op = target->next_op_to_issue;
while (curr_op != NULL) {
if (target->access_state == MPIDI_RMA_LOCK_ISSUED) {
/* It is possible that the previous OP+LOCK changes
* lock state to LOCK_ISSUED. */
break;
}
if (curr_op->next == NULL &&
target->sync.sync_flag == MPIDI_RMA_SYNC_NONE && curr_op->ureq == NULL) {
/* Skip the last OP if sync_flag is NONE since we
* want to leave it to the ending synchronization
* so that we can piggyback LOCK / FLUSH.
* However, if it is a request-based RMA, do not
* skip it (otherwise a wait call before unlock
* will be blocked). */
break;
}
flags = MPIDI_CH3_PKT_FLAG_NONE;
if (first_op) {
/* piggyback on first OP. */
if (target->access_state == MPIDI_RMA_LOCK_CALLED) {
MPIU_Assert(curr_op->piggyback_lock_candidate);
if (target->lock_type == MPI_LOCK_SHARED)
flags |= MPIDI_CH3_PKT_FLAG_RMA_LOCK_SHARED;
else {
MPIU_Assert(target->lock_type == MPI_LOCK_EXCLUSIVE);
flags |= MPIDI_CH3_PKT_FLAG_RMA_LOCK_EXCLUSIVE;
}
target->access_state = MPIDI_RMA_LOCK_ISSUED;
}
first_op = 0;
}
if (curr_op->next == NULL) {
/* piggyback on last OP. */
if (target->sync.sync_flag == MPIDI_RMA_SYNC_FLUSH) {
flags |= MPIDI_CH3_PKT_FLAG_RMA_FLUSH;
if (target->win_complete_flag)
flags |= MPIDI_CH3_PKT_FLAG_RMA_DECR_AT_COUNTER;
}
else if (target->sync.sync_flag == MPIDI_RMA_SYNC_UNLOCK) {
flags |= MPIDI_CH3_PKT_FLAG_RMA_UNLOCK;
}
}
mpi_errno = issue_rma_op(curr_op, win_ptr, target, flags);
if (mpi_errno != MPI_SUCCESS)
MPIU_ERR_POP(mpi_errno);
(*made_progress) = 1;
if (curr_op->pkt.type == MPIDI_CH3_PKT_PUT ||
curr_op->pkt.type == MPIDI_CH3_PKT_PUT_IMMED ||
curr_op->pkt.type == MPIDI_CH3_PKT_ACCUMULATE ||
curr_op->pkt.type == MPIDI_CH3_PKT_ACCUMULATE_IMMED) {
target->put_acc_issued = 1; /* set PUT_ACC_FLAG when sending
* PUT/ACC operation. */
}
if ((curr_op->pkt.type == MPIDI_CH3_PKT_ACCUMULATE ||
curr_op->pkt.type == MPIDI_CH3_PKT_GET_ACCUM) && curr_op->issued_stream_count > 0) {
/* For ACC-like operations, if not all stream units
* are issued out, we stick to the current operation,
* otherwise we move on to the next operation. */
target->next_op_to_issue = curr_op;
}
else
target->next_op_to_issue = curr_op->next;
if (target->next_op_to_issue == NULL) {
if (flags & MPIDI_CH3_PKT_FLAG_RMA_FLUSH || flags & MPIDI_CH3_PKT_FLAG_RMA_UNLOCK) {
/* We are done with ending sync, unset target's sync_flag. */
target->sync.sync_flag = MPIDI_RMA_SYNC_NONE;
}
}
if (flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_SHARED ||
flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_EXCLUSIVE) {
/* If this operation is piggybacked with LOCK,
* do not move it out of pending list, and do
* not complete the user request, because we
* may need to re-transmit it. */
break;
}
if (curr_op->ureq != NULL) {
mpi_errno = set_user_req_after_issuing_op(curr_op);
if (mpi_errno != MPI_SUCCESS)
MPIU_ERR_POP(mpi_errno);
}
if (curr_op->reqs_size == 0) {
MPIU_Assert(curr_op->reqs == NULL);
/* Sending is completed immediately. */
MPIDI_CH3I_RMA_Ops_free_elem(win_ptr, &(target->pending_op_list),
&(target->pending_op_list_tail), curr_op);
}
else {
/* Sending is not completed immediately. */
MPIDI_CH3I_RMA_Ops_unlink(&(target->pending_op_list),
&(target->pending_op_list_tail), curr_op);
if (curr_op->is_dt) {
MPIDI_CH3I_RMA_Ops_append(&(target->dt_op_list),
&(target->dt_op_list_tail), curr_op);
}
else if (curr_op->pkt.type == MPIDI_CH3_PKT_PUT ||
curr_op->pkt.type == MPIDI_CH3_PKT_PUT_IMMED ||
curr_op->pkt.type == MPIDI_CH3_PKT_ACCUMULATE ||
curr_op->pkt.type == MPIDI_CH3_PKT_ACCUMULATE_IMMED) {
MPIDI_CH3I_RMA_Ops_append(&(target->write_op_list),
&(target->write_op_list_tail), curr_op);
}
else {
MPIDI_CH3I_RMA_Ops_append(&(target->read_op_list),
&(target->read_op_list_tail), curr_op);
}
}
curr_op = target->next_op_to_issue;
} /* end of while loop */
fn_exit:
return mpi_errno;
fn_fail:
goto fn_exit;
}
