static inline void
mx_test_or_wait(int blocking, mx_endpoint_t ep, mx_request_t *req,
int timo, mx_status_t *stat, uint32_t *result)
{
if (blocking)
mx_wait(ep, req, timo, stat, result);
else {
do {
mx_test(ep, req, stat, result);
} while(!*result);
}
}
int ompi_mtl_mx_progress( void ) {
mx_return_t mx_return;
mx_request_t mx_request;
mx_status_t mx_status;
uint32_t result;
mca_mtl_mx_request_t* mtl_mx_request;
int completed = 0;
while(1){
mx_return = mx_ipeek(ompi_mtl_mx.mx_endpoint,
&mx_request,
&result);
if( OPAL_UNLIKELY(mx_return != MX_SUCCESS) ) {
opal_output(ompi_mtl_base_framework.framework_output, "Error in mx_ipeek (error %s)\n", mx_strerror(mx_return));
}
if(result) {
completed++;
mx_return = mx_test(ompi_mtl_mx.mx_endpoint,
&mx_request,
&mx_status,
&result);
if( OPAL_UNLIKELY(mx_return != MX_SUCCESS) ) {
opal_output(ompi_mtl_base_framework.framework_output, "Error in mx_test (error %s)\n", mx_strerror(mx_return));
abort();
}
if( OPAL_UNLIKELY(0 == result) ) {
opal_output(ompi_mtl_base_framework.framework_output, "Error in ompi_mtl_mx_progress, mx_ipeek returned a request, mx_test on the request resulted failure.\n");
abort();
}
mtl_mx_request = (mca_mtl_mx_request_t*) mx_status.context;
if(OMPI_MTL_MX_ISEND == mtl_mx_request->type) {
if(mtl_mx_request->free_after) {
free(mtl_mx_request->mx_segment[0].segment_ptr);
}
} else {
assert( OMPI_MTL_MX_IRECV == mtl_mx_request->type );
ompi_mtl_datatype_unpack(mtl_mx_request->convertor,
mtl_mx_request->mx_segment[0].segment_ptr,
mx_status.xfer_length);
/* set the status */
MX_GET_SRC(mx_status.match_info,
mtl_mx_request->super.ompi_req->req_status.MPI_SOURCE);
MX_GET_TAG(mx_status.match_info,
mtl_mx_request->super.ompi_req->req_status.MPI_TAG);
mtl_mx_request->super.ompi_req->req_status._ucount =
mx_status.xfer_length;
}
/* suppose everything went just fine ... */
mtl_mx_request->super.ompi_req->req_status.MPI_ERROR = OMPI_SUCCESS;
if( OPAL_UNLIKELY(MX_STATUS_SUCCESS != mx_status.code) ) {
if( MX_STATUS_TRUNCATED == mx_status.code ) {
mtl_mx_request->super.ompi_req->req_status.MPI_ERROR = MPI_ERR_TRUNCATE;
} else {
mtl_mx_request->super.ompi_req->req_status.MPI_ERROR = MPI_ERR_INTERN;
}
return completed;
}
mtl_mx_request->super.completion_callback(&mtl_mx_request->super);
return completed;
} else {
return completed;
}
}
}
int mca_btl_mx_send( struct mca_btl_base_module_t* btl,
struct mca_btl_base_endpoint_t* endpoint,
struct mca_btl_base_descriptor_t* descriptor,
mca_btl_base_tag_t tag )
{
mca_btl_mx_module_t* mx_btl = (mca_btl_mx_module_t*)btl;
mca_btl_mx_frag_t* frag = (mca_btl_mx_frag_t*)descriptor;
mx_segment_t mx_segment[2];
mx_return_t mx_return;
uint64_t total_length = 0, tag64;
uint32_t i = 0;
int btl_ownership = (descriptor->des_flags & MCA_BTL_DES_FLAGS_BTL_OWNERSHIP);
if( OPAL_UNLIKELY(MCA_BTL_MX_CONNECTED != ((mca_btl_mx_endpoint_t*)endpoint)->status) ) {
if( MCA_BTL_MX_NOT_REACHEABLE == ((mca_btl_mx_endpoint_t*)endpoint)->status )
return OMPI_ERROR;
if( MCA_BTL_MX_CONNECTION_PENDING == ((mca_btl_mx_endpoint_t*)endpoint)->status )
return OMPI_ERR_OUT_OF_RESOURCE;
if( OMPI_SUCCESS != mca_btl_mx_proc_connect( (mca_btl_mx_endpoint_t*)endpoint ) )
return OMPI_ERROR;
}
frag->endpoint = endpoint;
frag->type = MCA_BTL_MX_SEND;
do {
mx_segment[i].segment_ptr = descriptor->des_src[i].seg_addr.pval;
mx_segment[i].segment_length = descriptor->des_src[i].seg_len;
total_length += descriptor->des_src[i].seg_len;
} while (++i < descriptor->des_src_cnt);
tag64 = 0x01ULL | (((uint64_t)tag) << 8);
mx_return = mx_isend( mx_btl->mx_endpoint, mx_segment, descriptor->des_src_cnt,
endpoint->mx_peer_addr,
tag64, frag, &frag->mx_request );
if( OPAL_UNLIKELY(MX_SUCCESS != mx_return) ) {
opal_output( 0, "mx_isend fails with error %s\n", mx_strerror(mx_return) );
return OMPI_ERROR;
}
#ifdef HAVE_MX_FORGET
{
uint32_t mx_result;
mx_return = mx_ibuffered( mx_btl->mx_endpoint, &(frag->mx_request), &mx_result );
if( OPAL_UNLIKELY(MX_SUCCESS != mx_return) ) {
opal_output( 0, "mx_ibuffered failed with error %d (%s)\n",
mx_return, mx_strerror(mx_return) );
frag->base.des_flags |= MCA_BTL_DES_SEND_ALWAYS_CALLBACK;
return OMPI_ERROR;
}
if( mx_result ) {
mx_return = mx_forget( mx_btl->mx_endpoint, &(frag->mx_request) );
if( OPAL_UNLIKELY(MX_SUCCESS != mx_return) ) {
opal_output( 0, "mx_forget failed with error %d (%s)\n",
mx_return, mx_strerror(mx_return) );
frag->base.des_flags |= MCA_BTL_DES_SEND_ALWAYS_CALLBACK;
return OMPI_SUCCESS;
}
if( MCA_BTL_DES_SEND_ALWAYS_CALLBACK & frag->base.des_flags ) {
frag->base.des_cbfunc( &(mx_btl->super), frag->endpoint,
&(frag->base), OMPI_SUCCESS);
}
if( btl_ownership ) {
MCA_BTL_MX_FRAG_RETURN( mx_btl, frag );
}
return 1;
}
}
#endif
if( 2048 > total_length ) {
mx_status_t mx_status;
uint32_t mx_result;
/* let's check for completness */
mx_return = mx_test( mx_btl->mx_endpoint, &(frag->mx_request),
&mx_status, &mx_result );
if( OPAL_LIKELY(MX_SUCCESS == mx_return) ) {
if( mx_result ) {
if( MCA_BTL_DES_SEND_ALWAYS_CALLBACK & frag->base.des_flags ) {
frag->base.des_cbfunc( &(mx_btl->super), frag->endpoint,
&(frag->base), OMPI_SUCCESS);
}
if( btl_ownership ) {
MCA_BTL_MX_FRAG_RETURN( mx_btl, frag );
}
return 1;
}
}
}
frag->base.des_flags |= MCA_BTL_DES_SEND_ALWAYS_CALLBACK;
return OMPI_SUCCESS;
}
int
ompi_mtl_mx_send(struct mca_mtl_base_module_t* mtl,
struct ompi_communicator_t* comm,
int dest,
int tag,
struct opal_convertor_t *convertor,
mca_pml_base_send_mode_t mode)
{
mx_return_t mx_return;
uint64_t match_bits;
mca_mtl_mx_request_t mtl_mx_request;
size_t length;
mx_status_t mx_status;
uint32_t result;
ompi_proc_t* ompi_proc = ompi_comm_peer_lookup( comm, dest );
mca_mtl_mx_endpoint_t* mx_endpoint = (mca_mtl_mx_endpoint_t*) ompi_proc->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_MTL];
char* where;
assert(mtl == &ompi_mtl_mx.super);
MX_SET_SEND_BITS(match_bits, comm->c_contextid, comm->c_my_rank, tag);
ompi_mtl_datatype_pack(convertor,
&mtl_mx_request.mx_segment[0].segment_ptr,
&length,
&mtl_mx_request.free_after);
mtl_mx_request.mx_segment[0].segment_length = length;
mtl_mx_request.convertor = convertor;
mtl_mx_request.type = OMPI_MTL_MX_ISEND;
OPAL_OUTPUT_VERBOSE((50, ompi_mtl_base_framework.framework_output,
"issend bits: 0x%016" PRIu64 "\n",
match_bits));
if(mode == MCA_PML_BASE_SEND_SYNCHRONOUS) {
mx_return = mx_issend( ompi_mtl_mx.mx_endpoint,
mtl_mx_request.mx_segment,
1,
mx_endpoint->mx_peer_addr,
match_bits,
&mtl_mx_request,
&mtl_mx_request.mx_request
);
where = "mx_issend";
} else {
mx_return = mx_isend( ompi_mtl_mx.mx_endpoint,
mtl_mx_request.mx_segment,
1,
mx_endpoint->mx_peer_addr,
match_bits,
&mtl_mx_request,
&mtl_mx_request.mx_request
);
where = "mx_isend";
}
if( OPAL_UNLIKELY(mx_return != MX_SUCCESS) ) {
char peer_name[MX_MAX_HOSTNAME_LEN];
if(MX_SUCCESS != mx_nic_id_to_hostname( mx_endpoint->mx_peer->nic_id, peer_name)) {
sprintf( peer_name, "unknown %lx nic_id", (long)mx_endpoint->mx_peer->nic_id );
}
opal_output(ompi_mtl_base_framework.framework_output, "Error in %s (error %s) sending to %s\n",
where, mx_strerror(mx_return), peer_name);
/* Free buffer if needed */
if(mtl_mx_request.free_after) {
free(mtl_mx_request.mx_segment[0].segment_ptr);
}
return OMPI_ERROR;
}
do {
mx_return = mx_test(ompi_mtl_mx.mx_endpoint,
&mtl_mx_request.mx_request,
&mx_status,
&result);
if( OPAL_UNLIKELY(mx_return != MX_SUCCESS) ) {
opal_output(ompi_mtl_base_framework.framework_output, "Error in mx_wait (error %s)\n", mx_strerror(mx_return));
abort();
}
if( OPAL_UNLIKELY(result && mx_status.code != MX_STATUS_SUCCESS) ) {
opal_output(ompi_mtl_base_framework.framework_output,
"Error in ompi_mtl_mx_send, mx_wait returned something other than MX_STATUS_SUCCESS: mx_status.code = %d.\n",
mx_status.code);
abort();
}
} while(!result);
/* Free buffer if needed */
if(mtl_mx_request.free_after) {
free(mtl_mx_request.mx_segment[0].segment_ptr);
}
return OMPI_SUCCESS;
}