int MPIR_Test_impl(MPIR_Request * request_ptr, int *flag, MPI_Status * status)
{
int mpi_errno = MPI_SUCCESS;
mpi_errno = MPID_Progress_test();
if (mpi_errno)
MPIR_ERR_POP(mpi_errno);
if (MPIR_Request_has_poll_fn(request_ptr)) {
mpi_errno = MPIR_Grequest_poll(request_ptr, status);
if (mpi_errno)
MPIR_ERR_POP(mpi_errno);
}
if (MPIR_Request_is_complete(request_ptr))
*flag = TRUE;
else
*flag = FALSE;
fn_exit:
return mpi_errno;
fn_fail:
goto fn_exit;
}
int MPIR_Testsome_impl(int incount, MPIR_Request * request_ptrs[],
int *outcount, int array_of_indices[], MPI_Status array_of_statuses[])
{
int i;
int n_inactive;
int mpi_errno = MPI_SUCCESS;
mpi_errno = MPID_Progress_test();
/* --BEGIN ERROR HANDLING-- */
if (mpi_errno != MPI_SUCCESS)
MPIR_ERR_POP(mpi_errno);
/* --END ERROR HANDLING-- */
n_inactive = 0;
*outcount = 0;
for (i = 0; i < incount; i++) {
if ((i + 1) % MPIR_CVAR_REQUEST_POLL_FREQ == 0) {
mpi_errno = MPID_Progress_test();
if (mpi_errno)
MPIR_ERR_POP(mpi_errno);
}
if (request_ptrs[i] != NULL && MPIR_Request_has_poll_fn(request_ptrs[i])) {
mpi_errno = MPIR_Grequest_poll(request_ptrs[i], &array_of_statuses[i]);
if (mpi_errno != MPI_SUCCESS)
goto fn_fail;
}
if (!MPIR_Request_is_active(request_ptrs[i])) {
n_inactive += 1;
} else if (MPIR_Request_is_complete(request_ptrs[i])) {
array_of_indices[*outcount] = i;
*outcount += 1;
}
}
if (n_inactive == incount)
*outcount = MPI_UNDEFINED;
fn_exit:
return mpi_errno;
fn_fail:
goto fn_exit;
}
int MPIR_Testany_impl(int count, MPIR_Request * request_ptrs[],
int *indx, int *flag, MPI_Status * status)
{
int i;
int n_inactive = 0;
int mpi_errno = MPI_SUCCESS;
mpi_errno = MPID_Progress_test();
/* --BEGIN ERROR HANDLING-- */
if (mpi_errno)
MPIR_ERR_POP(mpi_errno);
/* --END ERROR HANDLING-- */
for (i = 0; i < count; i++) {
if ((i + 1) % MPIR_CVAR_REQUEST_POLL_FREQ == 0) {
mpi_errno = MPID_Progress_test();
if (mpi_errno)
MPIR_ERR_POP(mpi_errno);
}
if (request_ptrs[i] != NULL && MPIR_Request_has_poll_fn(request_ptrs[i])) {
mpi_errno = MPIR_Grequest_poll(request_ptrs[i], status);
if (mpi_errno != MPI_SUCCESS)
goto fn_fail;
}
if (!MPIR_Request_is_active(request_ptrs[i])) {
n_inactive += 1;
} else if (MPIR_Request_is_complete(request_ptrs[i])) {
*flag = TRUE;
*indx = i;
goto fn_exit;
}
}
if (n_inactive == count) {
*flag = TRUE;
*indx = MPI_UNDEFINED;
}
fn_exit:
return mpi_errno;
fn_fail:
goto fn_exit;
}
static int MPIR_Bsend_check_active( void )
{
int mpi_errno = MPI_SUCCESS;
MPIR_Bsend_data_t *active = BsendBuffer.active, *next_active;
MPIU_DBG_MSG_P(BSEND,TYPICAL,"Checking active starting at %p", active);
while (active) {
MPI_Request r = active->request->handle;
int flag;
next_active = active->next;
if (active->kind == IBSEND) {
/* We handle ibsend specially to allow for the user
to attempt and cancel the request. Also, to allow
for a cancel attempt (which must be attempted before
a successful test or wait), we only start
testing when the user has successfully released
the request (it is a grequest, the free call will do it) */
flag = 0;
/* XXX DJG FIXME-MT should we be checking this? */
if (MPIU_Object_get_ref(active->request) == 1) {
mpi_errno = MPIR_Test_impl(&r, &flag, MPI_STATUS_IGNORE );
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
} else {
/* We need to invoke the progress engine in case we
need to advance other, incomplete communication. */
MPID_Progress_state progress_state;
MPID_Progress_start(&progress_state);
mpi_errno = MPID_Progress_test( );
MPID_Progress_end(&progress_state);
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
}
} else {
mpi_errno = MPIR_Test_impl( &r, &flag, MPI_STATUS_IGNORE );
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
}
if (flag) {
/* We're done. Remove this segment */
MPIU_DBG_MSG_P(BSEND,TYPICAL,"Removing segment %p", active);
MPIR_Bsend_free_segment( active );
}
active = next_active;
MPIU_DBG_MSG_P(BSEND,TYPICAL,"Next active is %p",active);
}
fn_exit:
return mpi_errno;
fn_fail:
goto fn_exit;
}
int MPIR_Test_impl(MPI_Request *request, int *flag, MPI_Status *status)
{
int mpi_errno = MPI_SUCCESS;
int active_flag;
MPID_Request *request_ptr = NULL;
/* If this is a null request handle, then return an empty status */
if (*request == MPI_REQUEST_NULL) {
MPIR_Status_set_empty(status);
*flag = TRUE;
goto fn_exit;
}
*flag = FALSE;
MPID_Request_get_ptr( *request, request_ptr );
/* If the request is already completed AND we want to avoid calling
the progress engine, we could make the call to MPID_Progress_test
conditional on the request not being completed. */
mpi_errno = MPID_Progress_test();
if (mpi_errno != MPI_SUCCESS) goto fn_fail;
if (request_ptr->kind == MPID_UREQUEST &&
request_ptr->greq_fns != NULL &&
request_ptr->greq_fns->poll_fn != NULL)
{
mpi_errno = (request_ptr->greq_fns->poll_fn)(request_ptr->greq_fns->grequest_extra_state, status);
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
}
if (MPID_Request_is_complete(request_ptr)) {
mpi_errno = MPIR_Request_complete(request, request_ptr, status,
&active_flag);
*flag = TRUE;
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
/* Fall through to the exit */
}
fn_exit:
return mpi_errno;
fn_fail:
goto fn_exit;
}
int MPID_Isend(const void * buf, int count, MPI_Datatype datatype, int rank,
int tag, MPID_Comm * comm, int context_offset,
MPID_Request ** request)
{
MPIDI_msg_sz_t data_sz;
int dt_contig;
MPI_Aint dt_true_lb;
MPID_Datatype * dt_ptr;
MPID_Request * sreq;
MPIDI_VC_t * vc=0;
#if defined(MPID_USE_SEQUENCE_NUMBERS)
MPID_Seqnum_t seqnum;
#endif
int mpi_errno = MPI_SUCCESS;
MPIDI_STATE_DECL(MPID_STATE_MPID_ISEND);
MPIDI_FUNC_ENTER(MPID_STATE_MPID_ISEND);
MPIU_DBG_MSG_FMT(CH3_OTHER,VERBOSE,(MPIU_DBG_FDEST,
"rank=%d, tag=%d, context=%d",
rank, tag, comm->context_id + context_offset));
if (rank == comm->rank && comm->comm_kind != MPID_INTERCOMM)
{
#if defined (_OSU_PSM_)
goto skip_self_send; /* psm will internally do self-send, no special
handling is needed here */
#endif /* _OSU_PSM_ */
mpi_errno = MPIDI_Isend_self(buf, count, datatype, rank, tag, comm,
context_offset, MPIDI_REQUEST_TYPE_SEND, &sreq);
goto fn_exit;
}
#if defined (_OSU_PSM_)
skip_self_send:
#endif
if (rank != MPI_PROC_NULL) {
MPIDI_Comm_get_vc_set_active(comm, rank, &vc);
#ifdef ENABLE_COMM_OVERRIDES
/* this needs to come before the sreq is created, since the override
* function is responsible for creating its own request */
if (vc->comm_ops && vc->comm_ops->isend)
{
mpi_errno = vc->comm_ops->isend( vc, buf, count, datatype, rank, tag, comm, context_offset, &sreq);
goto fn_exit;
}
#endif
}
MPIDI_Request_create_sreq(sreq, mpi_errno, goto fn_exit);
MPIDI_Request_set_type(sreq, MPIDI_REQUEST_TYPE_SEND);
if (rank == MPI_PROC_NULL)
{
MPIU_Object_set_ref(sreq, 1);
MPID_cc_set(&sreq->cc, 0);
goto fn_exit;
}
MPIDI_Datatype_get_info(count, datatype, dt_contig, data_sz, dt_ptr,
dt_true_lb);
if (data_sz == 0)
{
#if defined (_OSU_PSM_)
goto eager_send;
#endif /* _OSU_PSM_ */
MPIDI_CH3_Pkt_t upkt;
MPIDI_CH3_Pkt_eager_send_t * const eager_pkt = &upkt.eager_send;
MPIDI_Request_set_msg_type(sreq, MPIDI_REQUEST_EAGER_MSG);
sreq->dev.OnDataAvail = 0;
MPIU_DBG_MSG(CH3_OTHER,VERBOSE,"sending zero length message");
MPIDI_Pkt_init(eager_pkt, MPIDI_CH3_PKT_EAGER_SEND);
eager_pkt->match.parts.rank = comm->rank;
eager_pkt->match.parts.tag = tag;
eager_pkt->match.parts.context_id = comm->context_id + context_offset;
eager_pkt->sender_req_id = sreq->handle;
eager_pkt->data_sz = 0;
MPIDI_VC_FAI_send_seqnum(vc, seqnum);
MPIDI_Pkt_set_seqnum(eager_pkt, seqnum);
MPIDI_Request_set_seqnum(sreq, seqnum);
MPIU_THREAD_CS_ENTER(CH3COMM,vc);
mpi_errno = MPIU_CALL(MPIDI_CH3,iSend(vc, sreq, eager_pkt,
sizeof(*eager_pkt)));
MPIU_THREAD_CS_EXIT(CH3COMM,vc);
/* --BEGIN ERROR HANDLING-- */
if (mpi_errno != MPI_SUCCESS)
{
MPIU_Object_set_ref(sreq, 0);
MPIDI_CH3_Request_destroy(sreq);
sreq = NULL;
MPIU_ERR_SET(mpi_errno, MPI_ERR_OTHER, "**ch3|eagermsg");
goto fn_exit;
}
/* --END ERROR HANDLING-- */
goto fn_exit;
}
#if defined (_OSU_PSM_)
if(HANDLE_GET_KIND(datatype) != HANDLE_KIND_BUILTIN) {
sreq->dev.datatype_ptr = dt_ptr;
MPID_Datatype_add_ref(dt_ptr);
sreq->psm_flags |= PSM_NEED_DTYPE_RELEASE;
}
if(vc->force_eager)
goto eager_send;
#endif /* _OSU_PSM_ */
#if defined(_OSU_MVAPICH_)
int i;
for (i = 0 ; i < rdma_num_extra_polls; i++)
{
if (rdma_global_ext_sendq_size > 1)
MPID_Progress_test();
}
#endif
/* FIXME: flow control: limit number of outstanding eager messsages
containing data and need to be buffered by the receiver */
#if defined(_OSU_MVAPICH_)
if (data_sz + sizeof(MPIDI_CH3_Pkt_eager_send_t) <= vc->eager_max_msg_sz
&& !vc->force_rndv)
#else /* defined(_OSU_MVAPICH_) */
if (data_sz + sizeof(MPIDI_CH3_Pkt_eager_send_t) <= vc->eager_max_msg_sz)
#endif /* defined(_OSU_MVAPICH_) */
{
#if defined (_OSU_PSM_)
eager_send:
#endif /* _OSU_PSM */
if (dt_contig)
{
mpi_errno = MPIDI_CH3_EagerContigIsend( &sreq,
MPIDI_CH3_PKT_EAGER_SEND,
(char*)buf + dt_true_lb,
data_sz, rank, tag,
comm, context_offset );
}
else
{
#if defined (_OSU_PSM_)
sreq->psm_flags |= PSM_NON_BLOCKING_SEND;
#endif
mpi_errno = MPIDI_CH3_EagerNoncontigSend( &sreq,
MPIDI_CH3_PKT_EAGER_SEND,
buf, count, datatype,
data_sz, rank, tag,
comm, context_offset );
#if defined (_OSU_PSM_)
goto fn_exit;
#endif
/* If we're not complete, then add a reference to the datatype */
if (sreq && sreq->dev.OnDataAvail) {
sreq->dev.datatype_ptr = dt_ptr;
MPID_Datatype_add_ref(dt_ptr);
}
}
}
else
{
/* Note that the sreq was created above */
MPIDI_Request_set_msg_type( sreq, MPIDI_REQUEST_RNDV_MSG );
mpi_errno = vc->rndvSend_fn( &sreq, buf, count, datatype, dt_contig,
data_sz, dt_true_lb, rank, tag, comm,
context_offset );
/* FIXME: fill temporary IOV or pack temporary buffer after send to
hide some latency. This requires synchronization
because the CTS packet could arrive and be processed before the
above iStartmsg completes (depending on the progress
engine, threads, etc.). */
#if defined(_OSU_MVAPICH_)
/* rndv transfers need to process CTS packet to initiate the actual RDMA transfer */
MPID_Progress_test();
#endif /* defined(_OSU_MVAPICH_) */
if (sreq && dt_ptr != NULL)
{
sreq->dev.datatype_ptr = dt_ptr;
MPID_Datatype_add_ref(dt_ptr);
}
}
fn_exit:
*request = sreq;
#if defined(_OSU_MVAPICH_)
for (i = 0 ; i < rdma_num_extra_polls; i++)
{
if (rdma_global_ext_sendq_size > 1)
MPID_Progress_test();
}
#endif
MPIU_DBG_STMT(CH3_OTHER,VERBOSE,
{
if (sreq != NULL)
{
MPIU_DBG_MSG_P(CH3_OTHER,VERBOSE,"request allocated, handle=0x%08x", sreq->handle);
}
}
);
int MPIDI_CH3_Connect_to_root(const char* port_name, MPIDI_VC_t** new_vc)
{
int mpi_errno = MPI_SUCCESS;
int str_errno;
char ifname[MAX_HOST_DESCRIPTION_LEN];
MPIDI_VC_t *vc;
MPIDI_CH3_Pkt_cm_establish_t pkt;
MPID_Request * sreq;
int seqnum;
MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_CONNECT_TO_ROOT);
MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3_CONNECT_TO_ROOT);
*new_vc = NULL;
if (!MPIDI_CH3I_Process.has_dpm)
return MPIR_Err_create_code(MPI_SUCCESS, MPIR_ERR_FATAL, FCNAME,
__LINE__, MPI_ERR_OTHER, "**notimpl", 0);
str_errno = MPIU_Str_get_string_arg(port_name,
MPIDI_CH3I_HOST_DESCRIPTION_KEY,
ifname, MAX_HOST_DESCRIPTION_LEN);
if (str_errno != MPIU_STR_SUCCESS) {
/* --BEGIN ERROR HANDLING */
if (str_errno == MPIU_STR_FAIL) {
MPIU_ERR_SETANDJUMP(mpi_errno,MPI_ERR_OTHER,"**argstr_missinghost");
}
else {
/* MPIU_STR_TRUNCATED or MPIU_STR_NONEM */
MPIU_ERR_SETANDJUMP(mpi_errno,MPI_ERR_OTHER, "**argstr_hostd");
}
/* --END ERROR HANDLING-- */
}
vc = MPIU_Malloc(sizeof(MPIDI_VC_t));
if (!vc) {
MPIU_ERR_SETANDJUMP(mpi_errno,MPI_ERR_OTHER, "**nomem");
}
MPIDI_VC_Init(vc, NULL, 0);
mpi_errno = MPIDI_CH3I_CM_Connect_raw_vc(vc, ifname);
if (mpi_errno) {
MPIU_ERR_POP(mpi_errno);
}
while (vc->ch.state != MPIDI_CH3I_VC_STATE_IDLE) {
mpi_errno = MPID_Progress_test();
/* --BEGIN ERROR HANDLING-- */
if (mpi_errno != MPI_SUCCESS) {
MPIU_ERR_POP(mpi_errno);
}
}
/* fprintf(stderr, "[###] vc state to idel, now send cm_establish msg\n") */
/* Now a connection is created, send a cm_establish message */
/* FIXME: vc->mrail.remote_vc_addr is used to find remote vc
* A more elegant way is needed */
MPIDI_Pkt_init(&pkt, MPIDI_CH3_PKT_CM_ESTABLISH);
MPIDI_VC_FAI_send_seqnum(vc, seqnum);
MPIDI_Pkt_set_seqnum(&pkt, seqnum);
pkt.vc_addr = vc->mrail.remote_vc_addr;
mpi_errno = MPIDI_GetTagFromPort(port_name, &pkt.port_name_tag);
if (mpi_errno != MPIU_STR_SUCCESS) {
MPIU_ERR_SETANDJUMP(mpi_errno,MPI_ERR_OTHER, "**argstr_port_name_tag");
}
mpi_errno = MPIDI_CH3_iStartMsg(vc, &pkt, sizeof(pkt), &sreq);
if (mpi_errno != MPI_SUCCESS)
{
MPIU_ERR_SETANDJUMP1(mpi_errno,MPI_ERR_OTHER,"**fail", "**fail %s",
"Failed to send cm establish message");
}
if (sreq != NULL)
{
if (sreq->status.MPI_ERROR != MPI_SUCCESS)
{
mpi_errno = MPIR_Err_create_code(sreq->status.MPI_ERROR,
MPIR_ERR_FATAL, FCNAME, __LINE__,
MPI_ERR_OTHER,
"**fail", 0);
MPID_Request_release(sreq);
goto fn_fail;
}
MPID_Request_release(sreq);
}
*new_vc = vc;
fn_fail:
MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_CONNECT_TO_ROOT);
return mpi_errno;
}
