MPID_Request * MPIDI_CH3U_Recvq_FDU_or_AEP(int source, int tag,
int context_id, MPID_Comm *comm, void *user_buf,
int user_count, MPI_Datatype datatype, int * foundp)
{
MPID_Time_t timer_start;
int found;
MPID_Request *rreq, *prev_rreq;
MPIDI_Message_match match;
MPIDI_Message_match mask;
MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3U_RECVQ_FDU_OR_AEP);
MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3U_RECVQ_FDU_OR_AEP);
MPIU_THREAD_CS_ASSERT_HELD(MSGQUEUE);
/* Store how much time is spent traversing the queue */
MPIR_T_START_TIMER(RECVQ_STATISTICS, timer_start);
/* Optimize this loop for an empty unexpected receive queue */
rreq = recvq_unexpected_head;
if (rreq) {
prev_rreq = NULL;
match.parts.context_id = context_id;
match.parts.tag = tag;
match.parts.rank = source;
if (tag != MPI_ANY_TAG && source != MPI_ANY_SOURCE) {
do {
MPIR_T_INC(RECVQ_STATISTICS, unexpected_recvq_match_attempts);
if (MATCH_WITH_NO_MASK(rreq->dev.match, match)) {
if (prev_rreq != NULL) {
prev_rreq->dev.next = rreq->dev.next;
}
else {
recvq_unexpected_head = rreq->dev.next;
}
if (rreq->dev.next == NULL) {
recvq_unexpected_tail = prev_rreq;
}
MPIR_T_DEC(RECVQ_STATISTICS, unexpected_qlen);
if (MPIDI_Request_get_msg_type(rreq) == MPIDI_REQUEST_EAGER_MSG)
MPIR_T_SUBTRACT(RECVQ_STATISTICS, MPIDI_CH3I_unexpected_recvq_buffer_size, rreq->dev.tmpbuf_sz);
rreq->comm = comm;
MPIR_Comm_add_ref(comm);
rreq->dev.user_buf = user_buf;
rreq->dev.user_count = user_count;
rreq->dev.datatype = datatype;
found = TRUE;
goto lock_exit;
}
prev_rreq = rreq;
rreq = rreq->dev.next;
} while (rreq);
}
else {
mask.parts.context_id = mask.parts.rank = mask.parts.tag = ~0;
if (tag == MPI_ANY_TAG)
match.parts.tag = mask.parts.tag = 0;
if (source == MPI_ANY_SOURCE)
match.parts.rank = mask.parts.rank = 0;
do {
MPIR_T_INC(RECVQ_STATISTICS, unexpected_recvq_match_attempts);
if (MATCH_WITH_LEFT_MASK(rreq->dev.match, match, mask)) {
if (prev_rreq != NULL) {
prev_rreq->dev.next = rreq->dev.next;
}
else {
recvq_unexpected_head = rreq->dev.next;
}
if (rreq->dev.next == NULL) {
recvq_unexpected_tail = prev_rreq;
}
MPIR_T_DEC(RECVQ_STATISTICS, unexpected_qlen);
if (MPIDI_Request_get_msg_type(rreq) == MPIDI_REQUEST_EAGER_MSG)
MPIR_T_SUBTRACT(RECVQ_STATISTICS, MPIDI_CH3I_unexpected_recvq_buffer_size, rreq->dev.tmpbuf_sz);
rreq->comm = comm;
MPIR_Comm_add_ref(comm);
rreq->dev.user_buf = user_buf;
rreq->dev.user_count = user_count;
rreq->dev.datatype = datatype;
found = TRUE;
goto lock_exit;
}
prev_rreq = rreq;
rreq = rreq->dev.next;
} while (rreq);
}
}
MPIR_T_END_TIMER(RECVQ_STATISTICS, timer_start, time_matching_unexpectedq);
/* A matching request was not found in the unexpected queue, so we
need to allocate a new request and add it to the posted queue */
{
int mpi_errno = MPI_SUCCESS;
found = FALSE;
MPIDI_Request_create_rreq( rreq, mpi_errno, goto lock_exit );
rreq->dev.match.parts.tag = tag;
rreq->dev.match.parts.rank = source;
rreq->dev.match.parts.context_id = context_id;
/* Added a mask for faster search on 64-bit capable
* platforms */
rreq->dev.mask.parts.context_id = ~0;
if (rreq->dev.match.parts.rank == MPI_ANY_SOURCE)
rreq->dev.mask.parts.rank = 0;
else
rreq->dev.mask.parts.rank = ~0;
if (rreq->dev.match.parts.tag == MPI_ANY_TAG)
rreq->dev.mask.parts.tag = 0;
else
rreq->dev.mask.parts.tag = ~0;
rreq->comm = comm;
MPIR_Comm_add_ref(comm);
rreq->dev.user_buf = user_buf;
rreq->dev.user_count = user_count;
rreq->dev.datatype = datatype;
/* check whether VC has failed, or this is an ANY_SOURCE in a
failed communicator */
if (source != MPI_ANY_SOURCE) {
MPIDI_VC_t *vc;
MPIDI_Comm_get_vc(comm, source, &vc);
if (vc->state == MPIDI_VC_STATE_MORIBUND) {
MPIU_ERR_SET1(mpi_errno, MPIX_ERR_PROC_FAIL_STOP, "**comm_fail", "**comm_fail %d", vc->pg_rank);
rreq->status.MPI_ERROR = mpi_errno;
MPIDI_CH3U_Request_complete(rreq);
goto lock_exit;
}
} else if (!MPIDI_CH3I_Comm_AS_enabled(comm)) {
MPIU_ERR_SET(mpi_errno, MPIX_ERR_PROC_FAIL_STOP, "**comm_fail");
rreq->status.MPI_ERROR = mpi_errno;
MPIDI_CH3U_Request_complete(rreq);
goto lock_exit;
}
rreq->dev.next = NULL;
if (recvq_posted_tail != NULL) {
recvq_posted_tail->dev.next = rreq;
}
else {
recvq_posted_head = rreq;
}
recvq_posted_tail = rreq;
MPIR_T_INC(RECVQ_STATISTICS, posted_qlen);
MPIDI_POSTED_RECV_ENQUEUE_HOOK(rreq);
}
lock_exit:
*foundp = found;
/* If a match was not found, the timer was stopped after the traversal */
if (found)
MPIR_T_END_TIMER(RECVQ_STATISTICS, timer_start, time_matching_unexpectedq);
MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3U_RECVQ_FDU_OR_AEP);
return rreq;
}
MPID_Request * MPIDI_CH3U_Recvq_FDU_or_AEP(int source, int tag,
int context_id, MPID_Comm *comm, void *user_buf,
int user_count, MPI_Datatype datatype, int * foundp)
{
int found;
MPID_Request *rreq, *prev_rreq;
MPIDI_Message_match match;
MPIDI_Message_match mask;
MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3U_RECVQ_FDU_OR_AEP);
MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3U_RECVQ_FDU_OR_AEP);
MPIU_THREAD_CS_ASSERT_HELD(MSGQUEUE);
/* Optimize this loop for an empty unexpected receive queue */
rreq = recvq_unexpected_head;
if (rreq) {
prev_rreq = NULL;
match.parts.context_id = context_id;
match.parts.tag = tag;
match.parts.rank = source;
if (tag != MPI_ANY_TAG && source != MPI_ANY_SOURCE) {
do {
if (MATCH_WITH_NO_MASK(rreq->dev.match, match)) {
if (prev_rreq != NULL) {
prev_rreq->dev.next = rreq->dev.next;
}
else {
recvq_unexpected_head = rreq->dev.next;
}
if (rreq->dev.next == NULL) {
recvq_unexpected_tail = prev_rreq;
}
rreq->comm = comm;
MPIR_Comm_add_ref(comm);
rreq->dev.user_buf = user_buf;
rreq->dev.user_count = user_count;
rreq->dev.datatype = datatype;
found = TRUE;
goto lock_exit;
}
prev_rreq = rreq;
rreq = rreq->dev.next;
} while (rreq);
}
else {
mask.parts.context_id = mask.parts.rank = mask.parts.tag = ~0;
if (tag == MPI_ANY_TAG)
match.parts.tag = mask.parts.tag = 0;
if (source == MPI_ANY_SOURCE)
match.parts.rank = mask.parts.rank = 0;
do {
if (MATCH_WITH_LEFT_MASK(rreq->dev.match, match, mask)) {
if (prev_rreq != NULL) {
prev_rreq->dev.next = rreq->dev.next;
}
else {
recvq_unexpected_head = rreq->dev.next;
}
if (rreq->dev.next == NULL) {
recvq_unexpected_tail = prev_rreq;
}
rreq->comm = comm;
MPIR_Comm_add_ref(comm);
rreq->dev.user_buf = user_buf;
rreq->dev.user_count = user_count;
rreq->dev.datatype = datatype;
found = TRUE;
goto lock_exit;
}
prev_rreq = rreq;
rreq = rreq->dev.next;
} while (rreq);
}
}
/* A matching request was not found in the unexpected queue, so we
need to allocate a new request and add it to the posted queue */
{
int mpi_errno = MPI_SUCCESS;
found = FALSE;
MPIDI_Request_create_rreq( rreq, mpi_errno, goto lock_exit );
rreq->dev.match.parts.tag = tag;
rreq->dev.match.parts.rank = source;
rreq->dev.match.parts.context_id = context_id;
/* Added a mask for faster search on 64-bit capable
* platforms */
rreq->dev.mask.parts.context_id = ~0;
if (rreq->dev.match.parts.rank == MPI_ANY_SOURCE)
rreq->dev.mask.parts.rank = 0;
else
rreq->dev.mask.parts.rank = ~0;
if (rreq->dev.match.parts.tag == MPI_ANY_TAG)
rreq->dev.mask.parts.tag = 0;
else
rreq->dev.mask.parts.tag = ~0;
rreq->comm = comm;
MPIR_Comm_add_ref(comm);
rreq->dev.user_buf = user_buf;
rreq->dev.user_count = user_count;
rreq->dev.datatype = datatype;
/* check whether VC has failed, or this is an ANY_SOURCE in a
failed communicator */
if (source != MPI_ANY_SOURCE) {
MPIDI_VC_t *vc;
MPIDI_Comm_get_vc(comm, source, &vc);
if (vc->state == MPIDI_VC_STATE_MORIBUND) {
MPIU_ERR_SET1(mpi_errno, MPI_ERR_OTHER, "**comm_fail", "**comm_fail %d", vc->pg_rank);
rreq->status.MPI_ERROR = mpi_errno;
MPIDI_CH3U_Request_complete(rreq);
goto lock_exit;
}
} else if (MPID_VCRT_Contains_failed_vc(comm->vcrt)) {
MPIU_ERR_SET(mpi_errno, MPI_ERR_OTHER, "**comm_fail");
rreq->status.MPI_ERROR = mpi_errno;
MPIDI_CH3U_Request_complete(rreq);
goto lock_exit;
}
rreq->dev.next = NULL;
if (recvq_posted_tail != NULL) {
recvq_posted_tail->dev.next = rreq;
}
else {
recvq_posted_head = rreq;
}
recvq_posted_tail = rreq;
MPIDI_POSTED_RECV_ENQUEUE_HOOK(rreq);
}
lock_exit:
*foundp = found;
MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3U_RECVQ_FDU_OR_AEP);
return rreq;
}
