int MPIDI_CH3U_Recvq_FU(int source, int tag, int context_id, MPI_Status *s)
{
MPID_Request * rreq;
int found = 0;
MPIDI_Message_match match, mask;
MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3U_RECVQ_FU);
MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3U_RECVQ_FU);
MPIU_THREAD_CS_ASSERT_HELD(MSGQUEUE);
rreq = recvq_unexpected_head;
match.parts.context_id = context_id;
match.parts.tag = tag;
match.parts.rank = source;
if (tag != MPI_ANY_TAG && source != MPI_ANY_SOURCE) {
while (rreq != NULL) {
if (MATCH_WITH_NO_MASK(rreq->dev.match, match))
break;
rreq = rreq->dev.next;
}
}
else {
mask.parts.context_id = mask.parts.rank = mask.parts.tag = ~0;
if (tag == MPI_ANY_TAG)
match.parts.tag = mask.parts.tag = 0;
if (source == MPI_ANY_SOURCE)
match.parts.rank = mask.parts.rank = 0;
while (rreq != NULL) {
if (MATCH_WITH_LEFT_MASK(rreq->dev.match, match, mask))
break;
rreq = rreq->dev.next;
}
}
/* Save the information about the request before releasing the
queue */
if (rreq) {
if (s != MPI_STATUS_IGNORE) {
/* Avoid setting "extra" fields like MPI_ERROR */
s->MPI_SOURCE = rreq->status.MPI_SOURCE;
s->MPI_TAG = rreq->status.MPI_TAG;
s->count = rreq->status.count;
s->cancelled = rreq->status.cancelled;
}
found = 1;
}
MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3U_RECVQ_FU);
return found;
}
MPID_Request * MPIDI_CH3U_Recvq_FDU(MPI_Request sreq_id,
MPIDI_Message_match * match)
{
MPID_Request * rreq;
MPID_Request * prev_rreq;
MPID_Request * cur_rreq;
MPID_Request * matching_prev_rreq;
MPID_Request * matching_cur_rreq;
MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3U_RECVQ_FDU);
MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3U_RECVQ_FDU);
MPIU_THREAD_CS_ASSERT_HELD(MSGQUEUE);
matching_prev_rreq = NULL;
matching_cur_rreq = NULL;
prev_rreq = NULL;
/* Note that since this routine is used only in the case of send_cancel,
there can be only one match if at all. */
/* FIXME: Why doesn't this exit after it finds the first match? */
cur_rreq = recvq_unexpected_head;
while (cur_rreq != NULL) {
if (cur_rreq->dev.sender_req_id == sreq_id &&
(MATCH_WITH_NO_MASK(cur_rreq->dev.match, *match))) {
matching_prev_rreq = prev_rreq;
matching_cur_rreq = cur_rreq;
}
prev_rreq = cur_rreq;
cur_rreq = cur_rreq->dev.next;
}
if (matching_cur_rreq != NULL) {
if (matching_prev_rreq != NULL) {
matching_prev_rreq->dev.next = matching_cur_rreq->dev.next;
}
else {
recvq_unexpected_head = matching_cur_rreq->dev.next;
}
if (matching_cur_rreq->dev.next == NULL) {
recvq_unexpected_tail = matching_prev_rreq;
}
rreq = matching_cur_rreq;
}
else {
rreq = NULL;
}
MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3U_RECVQ_FDU);
return rreq;
}
int MPIDI_CH3U_Recvq_count_unexp(void)
{
int count = 0;
MPID_Request *req = recvq_unexpected_head;
MPIU_THREAD_CS_ASSERT_HELD(MSGQUEUE);
while (req)
{
++count;
req = req->dev.next;
}
return count;
}
MPID_Request * MPIDI_CH3U_Recvq_FDP_or_AEU(MPIDI_Message_match * match,
int * foundp)
{
MPID_Time_t timer_start;
int found;
MPID_Request * rreq;
MPID_Request * prev_rreq;
int channel_matched;
MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3U_RECVQ_FDP_OR_AEU);
MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3U_RECVQ_FDP_OR_AEU);
MPIU_THREAD_CS_ASSERT_HELD(MSGQUEUE);
top_loop:
prev_rreq = NULL;
rreq = recvq_posted_head;
MPIR_T_START_TIMER(RECVQ_STATISTICS, timer_start);
while (rreq != NULL) {
MPIR_T_INC(RECVQ_STATISTICS, posted_recvq_match_attempts);
if (MATCH_WITH_LEFT_RIGHT_MASK(rreq->dev.match, *match, rreq->dev.mask)) {
if (prev_rreq != NULL) {
prev_rreq->dev.next = rreq->dev.next;
}
else {
recvq_posted_head = rreq->dev.next;
}
if (rreq->dev.next == NULL) {
recvq_posted_tail = prev_rreq;
}
MPIR_T_DEC(RECVQ_STATISTICS, posted_qlen);
/* give channel a chance to match the request, try again if so */
channel_matched = MPIDI_POSTED_RECV_DEQUEUE_HOOK(rreq);
if (channel_matched)
goto top_loop;
found = TRUE;
goto lock_exit;
}
prev_rreq = rreq;
rreq = rreq->dev.next;
}
MPIR_T_END_TIMER(RECVQ_STATISTICS, timer_start, time_failed_matching_postedq);
/* A matching request was not found in the posted queue, so we
need to allocate a new request and add it to the unexpected queue */
{
int mpi_errno=0;
MPIDI_Request_create_rreq( rreq, mpi_errno,
found=FALSE;goto lock_exit );
MPIU_Assert(mpi_errno == 0);
rreq->dev.recv_pending_count = 1;
rreq->dev.match = *match;
rreq->dev.next = NULL;
if (recvq_unexpected_tail != NULL) {
recvq_unexpected_tail->dev.next = rreq;
}
else {
recvq_unexpected_head = rreq;
}
recvq_unexpected_tail = rreq;
MPIR_T_INC(RECVQ_STATISTICS, unexpected_qlen);
}
found = FALSE;
lock_exit:
*foundp = found;
MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3U_RECVQ_FDP_OR_AEU);
return rreq;
}
MPID_Request * MPIDI_CH3U_Recvq_FDU_or_AEP(int source, int tag,
int context_id, MPID_Comm *comm, void *user_buf,
int user_count, MPI_Datatype datatype, int * foundp)
{
MPID_Time_t timer_start;
int found;
MPID_Request *rreq, *prev_rreq;
MPIDI_Message_match match;
MPIDI_Message_match mask;
MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3U_RECVQ_FDU_OR_AEP);
MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3U_RECVQ_FDU_OR_AEP);
MPIU_THREAD_CS_ASSERT_HELD(MSGQUEUE);
/* Store how much time is spent traversing the queue */
MPIR_T_START_TIMER(RECVQ_STATISTICS, timer_start);
/* Optimize this loop for an empty unexpected receive queue */
rreq = recvq_unexpected_head;
if (rreq) {
prev_rreq = NULL;
match.parts.context_id = context_id;
match.parts.tag = tag;
match.parts.rank = source;
if (tag != MPI_ANY_TAG && source != MPI_ANY_SOURCE) {
do {
MPIR_T_INC(RECVQ_STATISTICS, unexpected_recvq_match_attempts);
if (MATCH_WITH_NO_MASK(rreq->dev.match, match)) {
if (prev_rreq != NULL) {
prev_rreq->dev.next = rreq->dev.next;
}
else {
recvq_unexpected_head = rreq->dev.next;
}
if (rreq->dev.next == NULL) {
recvq_unexpected_tail = prev_rreq;
}
MPIR_T_DEC(RECVQ_STATISTICS, unexpected_qlen);
if (MPIDI_Request_get_msg_type(rreq) == MPIDI_REQUEST_EAGER_MSG)
MPIR_T_SUBTRACT(RECVQ_STATISTICS, MPIDI_CH3I_unexpected_recvq_buffer_size, rreq->dev.tmpbuf_sz);
rreq->comm = comm;
MPIR_Comm_add_ref(comm);
rreq->dev.user_buf = user_buf;
rreq->dev.user_count = user_count;
rreq->dev.datatype = datatype;
found = TRUE;
goto lock_exit;
}
prev_rreq = rreq;
rreq = rreq->dev.next;
} while (rreq);
}
else {
mask.parts.context_id = mask.parts.rank = mask.parts.tag = ~0;
if (tag == MPI_ANY_TAG)
match.parts.tag = mask.parts.tag = 0;
if (source == MPI_ANY_SOURCE)
match.parts.rank = mask.parts.rank = 0;
do {
MPIR_T_INC(RECVQ_STATISTICS, unexpected_recvq_match_attempts);
if (MATCH_WITH_LEFT_MASK(rreq->dev.match, match, mask)) {
if (prev_rreq != NULL) {
prev_rreq->dev.next = rreq->dev.next;
}
else {
recvq_unexpected_head = rreq->dev.next;
}
if (rreq->dev.next == NULL) {
recvq_unexpected_tail = prev_rreq;
}
MPIR_T_DEC(RECVQ_STATISTICS, unexpected_qlen);
if (MPIDI_Request_get_msg_type(rreq) == MPIDI_REQUEST_EAGER_MSG)
MPIR_T_SUBTRACT(RECVQ_STATISTICS, MPIDI_CH3I_unexpected_recvq_buffer_size, rreq->dev.tmpbuf_sz);
rreq->comm = comm;
MPIR_Comm_add_ref(comm);
rreq->dev.user_buf = user_buf;
rreq->dev.user_count = user_count;
rreq->dev.datatype = datatype;
found = TRUE;
goto lock_exit;
}
prev_rreq = rreq;
rreq = rreq->dev.next;
} while (rreq);
}
}
MPIR_T_END_TIMER(RECVQ_STATISTICS, timer_start, time_matching_unexpectedq);
/* A matching request was not found in the unexpected queue, so we
need to allocate a new request and add it to the posted queue */
{
int mpi_errno = MPI_SUCCESS;
found = FALSE;
MPIDI_Request_create_rreq( rreq, mpi_errno, goto lock_exit );
rreq->dev.match.parts.tag = tag;
rreq->dev.match.parts.rank = source;
rreq->dev.match.parts.context_id = context_id;
/* Added a mask for faster search on 64-bit capable
* platforms */
rreq->dev.mask.parts.context_id = ~0;
if (rreq->dev.match.parts.rank == MPI_ANY_SOURCE)
rreq->dev.mask.parts.rank = 0;
else
rreq->dev.mask.parts.rank = ~0;
if (rreq->dev.match.parts.tag == MPI_ANY_TAG)
rreq->dev.mask.parts.tag = 0;
else
rreq->dev.mask.parts.tag = ~0;
rreq->comm = comm;
MPIR_Comm_add_ref(comm);
rreq->dev.user_buf = user_buf;
rreq->dev.user_count = user_count;
rreq->dev.datatype = datatype;
/* check whether VC has failed, or this is an ANY_SOURCE in a
failed communicator */
if (source != MPI_ANY_SOURCE) {
MPIDI_VC_t *vc;
MPIDI_Comm_get_vc(comm, source, &vc);
if (vc->state == MPIDI_VC_STATE_MORIBUND) {
MPIU_ERR_SET1(mpi_errno, MPIX_ERR_PROC_FAIL_STOP, "**comm_fail", "**comm_fail %d", vc->pg_rank);
rreq->status.MPI_ERROR = mpi_errno;
MPIDI_CH3U_Request_complete(rreq);
goto lock_exit;
}
} else if (!MPIDI_CH3I_Comm_AS_enabled(comm)) {
MPIU_ERR_SET(mpi_errno, MPIX_ERR_PROC_FAIL_STOP, "**comm_fail");
rreq->status.MPI_ERROR = mpi_errno;
MPIDI_CH3U_Request_complete(rreq);
goto lock_exit;
}
rreq->dev.next = NULL;
if (recvq_posted_tail != NULL) {
recvq_posted_tail->dev.next = rreq;
}
else {
recvq_posted_head = rreq;
}
recvq_posted_tail = rreq;
MPIR_T_INC(RECVQ_STATISTICS, posted_qlen);
MPIDI_POSTED_RECV_ENQUEUE_HOOK(rreq);
}
lock_exit:
*foundp = found;
/* If a match was not found, the timer was stopped after the traversal */
if (found)
MPIR_T_END_TIMER(RECVQ_STATISTICS, timer_start, time_matching_unexpectedq);
MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3U_RECVQ_FDU_OR_AEP);
return rreq;
}
MPID_Request * MPIDI_CH3U_Recvq_FDU_matchonly(int source, int tag, int context_id, MPID_Comm *comm, int *foundp)
{
MPID_Time_t timer_start;
int found = FALSE;
MPID_Request *rreq, *prev_rreq;
MPIDI_Message_match match;
MPIDI_Message_match mask;
MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3U_RECVQ_FDU_MATCHONLY);
MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3U_RECVQ_FDU_MATCHONLY);
MPIU_THREAD_CS_ASSERT_HELD(MSGQUEUE);
/* Store how much time is spent traversing the queue */
MPIR_T_START_TIMER(RECVQ_STATISTICS, timer_start);
/* Optimize this loop for an empty unexpected receive queue */
rreq = recvq_unexpected_head;
if (rreq) {
prev_rreq = NULL;
match.parts.context_id = context_id;
match.parts.tag = tag;
match.parts.rank = source;
if (tag != MPI_ANY_TAG && source != MPI_ANY_SOURCE) {
do {
MPIR_T_INC(RECVQ_STATISTICS, unexpected_recvq_match_attempts);
if (MATCH_WITH_NO_MASK(rreq->dev.match, match)) {
if (prev_rreq != NULL) {
prev_rreq->dev.next = rreq->dev.next;
}
else {
recvq_unexpected_head = rreq->dev.next;
}
if (rreq->dev.next == NULL) {
recvq_unexpected_tail = prev_rreq;
}
MPIR_T_DEC(RECVQ_STATISTICS, unexpected_qlen);
MPIR_T_SUBTRACT(RECVQ_STATISTICS, MPIDI_CH3I_unexpected_recvq_buffer_size, rreq->dev.tmpbuf_sz);
rreq->comm = comm;
MPIR_Comm_add_ref(comm);
/* don't have the (buf,count,type) info right now, can't add
* it to the request */
found = TRUE;
goto lock_exit;
}
prev_rreq = rreq;
rreq = rreq->dev.next;
} while (rreq);
}
else {
mask.parts.context_id = mask.parts.rank = mask.parts.tag = ~0;
if (tag == MPI_ANY_TAG)
match.parts.tag = mask.parts.tag = 0;
if (source == MPI_ANY_SOURCE)
match.parts.rank = mask.parts.rank = 0;
do {
MPIR_T_INC(RECVQ_STATISTICS, unexpected_recvq_match_attempts);
if (MATCH_WITH_LEFT_MASK(rreq->dev.match, match, mask)) {
if (prev_rreq != NULL) {
prev_rreq->dev.next = rreq->dev.next;
}
else {
recvq_unexpected_head = rreq->dev.next;
}
if (rreq->dev.next == NULL) {
recvq_unexpected_tail = prev_rreq;
}
MPIR_T_DEC(RECVQ_STATISTICS, unexpected_qlen);
MPIR_T_SUBTRACT(RECVQ_STATISTICS, MPIDI_CH3I_unexpected_recvq_buffer_size, rreq->dev.tmpbuf_sz);
rreq->comm = comm;
MPIR_Comm_add_ref(comm);
/* don't have the (buf,count,type) info right now, can't add
* it to the request */
found = TRUE;
goto lock_exit;
}
prev_rreq = rreq;
rreq = rreq->dev.next;
} while (rreq);
}
}
lock_exit:
MPIR_T_END_TIMER(RECVQ_STATISTICS, timer_start, time_matching_unexpectedq);
*foundp = found;
MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3U_RECVQ_FDU_MATCHONLY);
return rreq;
}
MPID_Request * MPIDI_CH3U_Recvq_FDU(MPI_Request sreq_id,
MPIDI_Message_match * match)
{
MPID_Time_t timer_start;
MPID_Request * rreq;
MPID_Request * prev_rreq;
MPID_Request * cur_rreq;
MPID_Request * matching_prev_rreq;
MPID_Request * matching_cur_rreq;
MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3U_RECVQ_FDU);
MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3U_RECVQ_FDU);
MPIU_THREAD_CS_ASSERT_HELD(MSGQUEUE);
matching_prev_rreq = NULL;
matching_cur_rreq = NULL;
prev_rreq = NULL;
/* Note that since this routine is used only in the case of send_cancel,
there can be only one match if at all. */
/* FIXME: Why doesn't this exit after it finds the first match? */
cur_rreq = recvq_unexpected_head;
while (cur_rreq != NULL) {
MPIR_T_START_TIMER(RECVQ_STATISTICS, timer_start);
if (cur_rreq->dev.sender_req_id == sreq_id) {
MPIR_T_INC(RECVQ_STATISTICS, unexpected_recvq_match_attempts);
if (MATCH_WITH_NO_MASK(cur_rreq->dev.match, *match)) {
matching_prev_rreq = prev_rreq;
matching_cur_rreq = cur_rreq;
}
}
MPIR_T_END_TIMER(RECVQ_STATISTICS, timer_start, time_matching_unexpectedq);
prev_rreq = cur_rreq;
cur_rreq = cur_rreq->dev.next;
}
if (matching_cur_rreq != NULL) {
if (matching_prev_rreq != NULL) {
matching_prev_rreq->dev.next = matching_cur_rreq->dev.next;
}
else {
recvq_unexpected_head = matching_cur_rreq->dev.next;
}
if (matching_cur_rreq->dev.next == NULL) {
recvq_unexpected_tail = matching_prev_rreq;
}
MPIR_T_DEC(RECVQ_STATISTICS, unexpected_qlen);
rreq = matching_cur_rreq;
MPIR_T_SUBTRACT(RECVQ_STATISTICS, MPIDI_CH3I_unexpected_recvq_buffer_size, rreq->dev.tmpbuf_sz);
}
else {
rreq = NULL;
}
MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3U_RECVQ_FDU);
return rreq;
}
MPID_Request * MPIDI_CH3U_Recvq_FDU_or_AEP(int source, int tag,
int context_id, MPID_Comm *comm, void *user_buf,
int user_count, MPI_Datatype datatype, int * foundp)
{
int found;
MPID_Request *rreq, *prev_rreq;
MPIDI_Message_match match;
MPIDI_Message_match mask;
MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3U_RECVQ_FDU_OR_AEP);
MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3U_RECVQ_FDU_OR_AEP);
MPIU_THREAD_CS_ASSERT_HELD(MSGQUEUE);
/* Optimize this loop for an empty unexpected receive queue */
rreq = recvq_unexpected_head;
if (rreq) {
prev_rreq = NULL;
match.parts.context_id = context_id;
match.parts.tag = tag;
match.parts.rank = source;
if (tag != MPI_ANY_TAG && source != MPI_ANY_SOURCE) {
do {
if (MATCH_WITH_NO_MASK(rreq->dev.match, match)) {
if (prev_rreq != NULL) {
prev_rreq->dev.next = rreq->dev.next;
}
else {
recvq_unexpected_head = rreq->dev.next;
}
if (rreq->dev.next == NULL) {
recvq_unexpected_tail = prev_rreq;
}
rreq->comm = comm;
MPIR_Comm_add_ref(comm);
rreq->dev.user_buf = user_buf;
rreq->dev.user_count = user_count;
rreq->dev.datatype = datatype;
found = TRUE;
goto lock_exit;
}
prev_rreq = rreq;
rreq = rreq->dev.next;
} while (rreq);
}
else {
mask.parts.context_id = mask.parts.rank = mask.parts.tag = ~0;
if (tag == MPI_ANY_TAG)
match.parts.tag = mask.parts.tag = 0;
if (source == MPI_ANY_SOURCE)
match.parts.rank = mask.parts.rank = 0;
do {
if (MATCH_WITH_LEFT_MASK(rreq->dev.match, match, mask)) {
if (prev_rreq != NULL) {
prev_rreq->dev.next = rreq->dev.next;
}
else {
recvq_unexpected_head = rreq->dev.next;
}
if (rreq->dev.next == NULL) {
recvq_unexpected_tail = prev_rreq;
}
rreq->comm = comm;
MPIR_Comm_add_ref(comm);
rreq->dev.user_buf = user_buf;
rreq->dev.user_count = user_count;
rreq->dev.datatype = datatype;
found = TRUE;
goto lock_exit;
}
prev_rreq = rreq;
rreq = rreq->dev.next;
} while (rreq);
}
}
/* A matching request was not found in the unexpected queue, so we
need to allocate a new request and add it to the posted queue */
{
int mpi_errno = MPI_SUCCESS;
found = FALSE;
MPIDI_Request_create_rreq( rreq, mpi_errno, goto lock_exit );
rreq->dev.match.parts.tag = tag;
rreq->dev.match.parts.rank = source;
rreq->dev.match.parts.context_id = context_id;
/* Added a mask for faster search on 64-bit capable
* platforms */
rreq->dev.mask.parts.context_id = ~0;
if (rreq->dev.match.parts.rank == MPI_ANY_SOURCE)
rreq->dev.mask.parts.rank = 0;
else
rreq->dev.mask.parts.rank = ~0;
if (rreq->dev.match.parts.tag == MPI_ANY_TAG)
rreq->dev.mask.parts.tag = 0;
else
rreq->dev.mask.parts.tag = ~0;
rreq->comm = comm;
MPIR_Comm_add_ref(comm);
rreq->dev.user_buf = user_buf;
rreq->dev.user_count = user_count;
rreq->dev.datatype = datatype;
/* check whether VC has failed, or this is an ANY_SOURCE in a
failed communicator */
if (source != MPI_ANY_SOURCE) {
MPIDI_VC_t *vc;
MPIDI_Comm_get_vc(comm, source, &vc);
if (vc->state == MPIDI_VC_STATE_MORIBUND) {
MPIU_ERR_SET1(mpi_errno, MPI_ERR_OTHER, "**comm_fail", "**comm_fail %d", vc->pg_rank);
rreq->status.MPI_ERROR = mpi_errno;
MPIDI_CH3U_Request_complete(rreq);
goto lock_exit;
}
} else if (MPID_VCRT_Contains_failed_vc(comm->vcrt)) {
MPIU_ERR_SET(mpi_errno, MPI_ERR_OTHER, "**comm_fail");
rreq->status.MPI_ERROR = mpi_errno;
MPIDI_CH3U_Request_complete(rreq);
goto lock_exit;
}
rreq->dev.next = NULL;
if (recvq_posted_tail != NULL) {
recvq_posted_tail->dev.next = rreq;
}
else {
recvq_posted_head = rreq;
}
recvq_posted_tail = rreq;
MPIDI_POSTED_RECV_ENQUEUE_HOOK(rreq);
}
lock_exit:
*foundp = found;
MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3U_RECVQ_FDU_OR_AEP);
return rreq;
}
